diff --git a/.travis.yml b/.travis.yml index e2291f64a77a7..19922a1bea19e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,11 @@ cache: # default timeout is too low timeout: 600 directories: + - $HOME/.rvm/ - $HOME/.m2 - $HOME/flink_cache + # keep in sync with tools/travis/setup_maven.sh + - $HOME/maven_cache # do not cache our own artifacts before_cache: @@ -41,60 +44,99 @@ env: global: # Global variable to avoid hanging travis builds when downloading cache archives. - MALLOC_ARENA_MAX=2 - # Build artifacts like logs (variables for apache/flink repo) - - secure: "gL3QRn6/XyVK+Em9RmVqpM6nbTwlhjK4/JiRYZGGCkBgTq4ZnG+Eq2qKAO22TAsqRSi7g7WAoAhUulPt0SJqH7hjMe0LetbO0izbVXDefwf2PJlsNgBbuFG6604++VUaUEyfPYYw9ADjV59LWG7+B/fjbRsevqRBZ30b1gv/tQQ=" - - secure: "eM9r8IglvnUKctxz/ga6hwGnCpdOvGyYdGj0H/UiNDEx3Lq1A6yp3gChEIXGJqRUXDI5TaIuidunUGY7KHml8urm8eG2Yk2ttxXehZqLpEaOU2jdNJCdLX8tlVfh14T9bxG5AYHQEV3qJUqDFtfXD3whvzuinrm1oEIA3qUxiA8=" - - secure: "EQYDWgJM5ANJ/sAFwmSEwSTOe9CDN/ENyQAr5/ntM67XanhTZj2Amgt9LthCRUU4EEPl/OFUTwNHMpv/+wa3q7dwVFldSIg5wyCndzJSATPyPBVjYgsXIQZVIjsq4TwTyrTteT55V6Oz2+l27Fvung2FPuN83ovswsJePFzMBxI=" - DOCKER_COMPOSE_VERSION=1.22.0 + - CHANGE_MINIKUBE_NONE_USER=true before_script: - "gem install --no-document --version 0.8.9 faraday " - "export -f travis_nanoseconds" - "export -f travis_time_start" - "export -f travis_time_finish" + # Download kubectl, which is a requirement for using minikube. + - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ + # Download minikube. + - curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ + - sudo minikube start --vm-driver=none --kubernetes-version=v1.9.0 + # Fix the kubectl context, as it's often stale. + - minikube update-context + # Wait for Kubernetes to be up and ready. + - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done # Install maven 3.2.5 since trusty uses 3.3.9 for which shading is broken before_install: - - "wget https://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.zip" - - "unzip -qq apache-maven-3.2.5-bin.zip" - - "rm apache-maven-3.2.5-bin.zip" - - "export M2_HOME=$PWD/apache-maven-3.2.5" - - "export PATH=$M2_HOME/bin:$PATH" - - "export MAVEN_OPTS=\"-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS\"" -# just in case: clean up the .m2 home and remove invalid jar files - - 'test ! -d $HOME/.m2/repository/ || find $HOME/.m2/repository/ -name "*.jar" -exec sh -c ''if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi'' \;' + - source ./tools/travis/setup_maven.sh # Installing the specified docker compose version - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin -# When modifying the matrix you also have to modify travis_controller.sh#getCurrentStage +notifications: + slack: + rooms: + - secure: ikPQn5JTpkyzxVyOPm/jIl3FPm6hY8xAdG4pSwxGWjBqF+NmmNTp9YZsJ6fD8xPql6T5n1hNDbZSC14jVUw/vvXGvibDXLN+06f25ZQl+4LJBXaiR7gTG6y3nO8G90Vw7XpvCme6n5Md9tvjygb17a4FEgRJFfwzWnnyPA1yvK0= + on_success: never + on_pull_requests: false + +stages: + - name: compile + - name: test + - name: E2E + if: type = cron + - name: cleanup + jdk: "oraclejdk8" jobs: include: # main profile - - stage: compile - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + stage: compile + script: ./tools/travis_controller.sh compile env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: compile - - stage: test - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + stage: test + script: ./tools/travis_controller.sh core env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: core - - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + script: ./tools/travis_controller.sh libraries env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: libraries - - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + script: ./tools/travis_controller.sh connectors env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: connectors - - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + script: ./tools/travis_controller.sh tests env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: tests - - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + script: ./tools/travis_controller.sh misc env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: misc - - stage: cleanup - script: ./tools/travis_controller.sh + - if: type in (pull_request, push) + stage: cleanup + script: ./tools/travis_controller.sh cleanup env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: cleanup + # E2E profile + - stage: E2E + env: PROFILE="-Dhadoop.version=2.8.3 -De2e-metrics" + script: ./tools/travis/nightly.sh split_misc.sh + name: misc - hadoop 2.8 + - env: PROFILE="-Dhadoop.version=2.8.3" + script: ./tools/travis/nightly.sh split_ha.sh + name: ha - hadoop 2.8 + - env: PROFILE="-Dhadoop.version=2.8.3" + script: ./tools/travis/nightly.sh split_sticky.sh + name: sticky - hadoop 2.8 + - env: PROFILE="-Dhadoop.version=2.8.3" + script: ./tools/travis/nightly.sh split_checkpoints.sh + name: checkpoints - hadoop 2.8 + - env: PROFILE="-Dhadoop.version=2.8.3" + script: ./tools/travis/nightly.sh split_container.sh + name: container - hadoop 2.8 + - env: PROFILE="-Dhadoop.version=2.8.3" + script: ./tools/travis/nightly.sh split_heavy.sh + name: heavy - hadoop 2.8 diff --git a/LICENSE b/LICENSE index 64850739a2535..261eeb9e9f8b2 100644 --- a/LICENSE +++ b/LICENSE @@ -199,124 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -======================================================================= - -Apache Flink subcomponents: - -The Apache Flink project contains subcomponents in the source code -release with separate copyright notices and license terms. Your use of -the source code for the these subcomponents is subject to the terms and -conditions of their respective licenses. - - ------------------------------------------------------------------------ - The MIT License ------------------------------------------------------------------------ - -The Apache Flink project bundles the following files under the MIT License: - - - Angular.JS v1.4.8 (http://angularjs.org) - Copyright (c) 2010-2015 Google, Inc. - - Angular Moment v0.10.3 (http://www.momentjs.com) - Copyright (C) 2013, 2014, 2015, Uri Shaked - - Angular UI Router v0.2.15 (http://angular-ui.github.io) - Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling - - angular-drag-and-drop-lists v1.4.0 (https://github.com/marceljuenemann/angular-drag-and-drop-lists) Copyright (c) 2014 Marcel Juenemann , Copyright (c) 2014-2016 Google Inc. - - Bootstrap v3.3.6 (http://getbootstrap.com) - Copyright (c) 2011-2015 Twitter, Inc - - d3 timeline (https://github.com/jiahuang/d3-timeline) - Copyright (c) 2012-2015 Jia Huang - - dagre v0.7.4 (https://github.com/cpettitt/dagre) - Copyright (c) 2012-2014 Chris Pettitt - - dagre-d3 v0.4.17 (https://github.com/cpettitt/dagre-d3) - Copyright (c) 2013 Chris Pettitt - - EvEmitter v1.0.2 (https://github.com/metafizzy/ev-emitter) - Copyright (C) 2016 David DeSandro - - Font Awesome (code) v4.5.0, v4.6.3 (http://fontawesome.io) - Copyright (c) 2014 Dave Gandy - - graphlib v1.0.7 (https://github.com/cpettitt/graphlib) - Copyright (c) 2012-2014 Chris Pettitt - - imagesloaded v4.1.0 (https://github.com/desandro/imagesloaded) - Copyright (C) 2016 David DeSandro - - JQuery v2.2.0 (http://jquery.com/) - Copyright 2014 jQuery Foundation and other contributors - - lodash v3.10.1 (http://dojofoundation.org) - Copyright 2012-2015 The Dojo Foundation - - moment.js v2.10.6 (http://momentjs.com/docs/) - Copyright (c) 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors - - moment-duration-format v1.3.0 (https://github.com/jsmreese/moment-duration-format) - Copyright (c) 2013 John Madhavan-Reese - - qTip2 v2.2.1 (http://qtip2.com) - Copyright (c) 2012 Craig Michael Thompson - - Split.js v1.0.6 (https://nathancahill.github.io/Split.js/) - Copyright (c) 2015 Nathan Cahill - -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - ------------------------------------------------------------------------ - BSD-style Licenses (BSD-3-clause) ------------------------------------------------------------------------ - -The Apache Flink project bundles the following files under BSD-3-clause licenses: - - D3 v3.5.12 (http://d3js.org/) - Copyright (c) 2010-2015, Michael Bostock - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list - of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - ------------------------------------------------------------------------ - The Open Font License ------------------------------------------------------------------------ - -The Apache Flink project bundles the following fonts under the -Open Font License (OFT) - http://scripts.sil.org/OFL - - - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy - -> fonts in "flink-runtime-web/web-dashboard/web/fonts" - -> fonts in "docs/page/font-awesome/fonts" - ------------------------------------------------------------------------ - The ISC License ------------------------------------------------------------------------ - -The Apache Flink project contains or reuses code that is licensed under the ISC license -from the following projects: - - - simplejmx (http://256stuff.com/sources/simplejmx/) Copyright (c) - Gray Watson - -Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby -granted, provided that this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING -ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE -USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/NOTICE b/NOTICE index da67d0e0d9e40..8fd6507657701 100644 --- a/NOTICE +++ b/NOTICE @@ -4,18 +4,52 @@ Copyright 2014-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -======================================================================= +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) -Apache Flink contains subcomponents with separate copyright notices and -license terms. Your use of the source code for the these subcomponents -is subject to the terms and conditions of their respective licenses. +- nvd3#1.8.4 -See the LICENSE file for a list of subcomponents and dependencies and -their respective licenses. +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) +See bundled license files for details. +- angular:1.4.8 +- angular-drag-and-drop-list:1.4.0 +- angular-moment:0.10.3 +- angular-ui-router:0.2.15 +- bootstrap:3.3.6 +- dagre:0.7.5 +- dagre-d3:0.4.17 +- ev-emitter:1.1.1 +- font-awesome:4.5.0 (CSS) +- graphlib:1.0.7 +- imagesloaded:4.1.4 +- jquery:2.2.0 +- lodash:3.10.1 +- moment:2.10.6 +- moment-duration-format:1.3.0 +- qtip2:2.2.1 +- Split.js:1.0.6 -lodash: ---------- -Copyright 2012-2015 The Dojo Foundation -Based on Underscore.js, copyright 2009-2015 Jeremy Ashkenas, -DocumentCloud and Investigative Reporters & Editors +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- d3:3.5.12 + +This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1). +See bundled license files for details. + +- font-awesome:4.5.0 (Font) (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy + -> fonts in "flink-runtime-web/web-dashboard/web/fonts" + -> fonts in "docs/page/font-awesome/fonts" + +The Apache Flink project contains or reuses code that is licensed under the ISC license from the following projects. + +- simplejmx (http://256stuff.com/sources/simplejmx/) Copyright (c) - Gray Watson + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby +granted, provided that this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING +ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE +USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/NOTICE-binary b/NOTICE-binary new file mode 100644 index 0000000000000..9028f72aaa4f3 --- /dev/null +++ b/NOTICE-binary @@ -0,0 +1,5863 @@ +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.chuusai:shapeless_2.11:2.3.2 +- log4j:log4j:1.2.17 +- net.sf.opencsv:opencsv:2.3 +- org.scalanlp:breeze_2.11:0.13 +- org.scalanlp:breeze-macros_2.11:0.13 +- org.typelevel:macro-compat_2.11:1.1.1 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.github.fommil.netlib:core:1.1.2 +- net.sourceforge.f2j:arpack_combined_all:0.1 + +This project bundles the following dependencies under the MIT/X11 license. +See bundled license files for details. + +- org.slf4j:slf4j-log4j12:1.7.15 +- org.spire-math:spire-macros_2.11:0.13.0 +- org.spire-math:spire_2.11:0.13.0 +- org.typelevel:machinist_2.11:0.6.1 + +This project bundles the following dependencies under the Mozilla Public license. + +- com.github.rwl:jtransforms:2.4.0 (https://sourceforge.net/projects/jtransforms/) + +flink-metrics-datadog +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.squareup.okhttp3:okhttp:3.7.0 +- com.squareup.okio:okio:1.12.0 + +flink-dist +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.code.findbugs:jsr305:1.3.9 +- com.twitter:chill-java:0.7.6 +- com.twitter:chill_2.11:0.7.6 +- com.typesafe:config:1.3.0 +- com.typesafe:ssl-config-core_2.11:0.2.1 +- com.typesafe.akka:akka-actor_2.11:2.4.20 +- com.typesafe.akka:akka-camel_2.11:2.4.20 +- com.typesafe.akka:akka-protobuf_2.11:2.4.20 +- com.typesafe.akka:akka-slf4j_2.11:2.4.20 +- com.typesafe.akka:akka-stream_2.11:2.4.20 +- commons-cli:commons-cli:1.3.1 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- org.apache.camel:camel-core:2.17.7 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-lang3:3.3.2 +- org.apache.commons:commons-math3:3.5 +- org.javassist:javassist:3.19.0-GA +- org.objenesis:objenesis:2.1 +- org.rocksdb:rocksdbjni:5.7.5 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.esotericsoftware.kryo:kryo:2.24.0 +- com.esotericsoftware.minlog:minlog:1.2 +- org.clapper:grizzled-slf4j_2.11:1.3.2 + +The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala. + +- org.scala-lang:scala-compiler:2.11.12 +- org.scala-lang:scala-library:2.11.12 +- org.scala-lang:scala-reflect:2.11.12 +- org.scala-lang.modules:scala-java8-compat_2.11:0.7.0 +- org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4 +- org.scala-lang.modules:scala-xml_2.11:1.0.5 + +This project bundles the following dependencies under the MIT/X11 license. +See bundled license files for details. + +- com.github.scopt:scopt_2.11:3.5.0 +- org.slf4j:slf4j-api:1.7.15 + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles the following dependencies under the Creative Commons CC0 "No Rights Reserved". + +- org.reactivestreams:reactive-streams:1.0.0 + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +flink-runtime +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.typesafe.akka:akka-remote_2.11:2.4.20 +- io.netty:netty:3.10.6.Final +- org.apache.zookeeper:zookeeper:3.4.10 +- org.uncommons.maths:uncommons-maths:1.2.2a + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +flink-shaded-curator +Copyright 2014-2018 The Apache Software Foundation + +- com.google.guava:guava:16.0.1 +- org.apache.curator:curator-client:2.12.0 +- org.apache.curator:curator-framework:2.12.0 +- org.apache.curator:curator-recipes:2.12.0 + +Curator Recipes +Copyright 2011-2017 The Apache Software Foundation + +Curator Framework +Copyright 2011-2017 The Apache Software Foundation + +Curator Client +Copyright 2011-2017 The Apache Software Foundation + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +flink-shaded-jackson-2 +Copyright 2014-2018 The Apache Software Foundation + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +flink-mesos +Copyright 2014-2018 The Apache Software Foundation + +- com.netflix.fenzo:fenzo-core:0.10.1 +- org.apache.mesos:mesos:1.0.1 +- com.fasterxml.jackson.core:jackson-annotations:2.4.0 +- com.fasterxml.jackson.core:jackson-core:2.4.5 +- com.fasterxml.jackson.core:jackson-databind:2.4.5 + +- com.google.protobuf:protobuf-java:2.6.2 + +mesos +Copyright 2016 The Apache Software Foundation + +flink-yarn +Copyright 2014-2018 The Apache Software Foundation + +Camel :: Core +Copyright 2007-2017 The Apache Software Foundation + + ========================================================================= + == NOTICE file corresponding to the section 4 d of == + == the Apache License, Version 2.0, == + == in this case for the Apache Camel distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Please read the different LICENSE files present in the licenses directory of + this distribution. + +Objenesis +Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita + +flink-table +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.esri.geometry:esri-geometry-api:2.0.0 +- com.google.guava:guava:19.0 +- joda-time:joda-time:2.5 +- net.hydromatic:aggdesigner-algorithm:6.0 +- org.apache.calcite:calcite-core:1.17.0 +- org.apache.calcite:calcite-linq4j:1.17.0 +- org.apache.calcite.avatica:avatica-core:1.12.0 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details + +- org.codehaus.janino:janino:3.0.7 +- org.codehaus.janino:commons-compiler:3.0.7 + +Calcite Core +Copyright 2012-2018 The Apache Software Foundation + +Apache Calcite Avatica +Copyright 2012-2018 The Apache Software Foundation + +Calcite Linq4j +Copyright 2012-2018 The Apache Software Foundation + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +flink-runtime-web +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- nvd3#1.8.4 + +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) +See bundled license files for details. + +- angular:1.4.8 +- angular-drag-and-drop-list:1.4.0 +- angular-moment:0.10.3 +- angular-ui-router:0.2.15 +- bootstrap:3.3.6 +- dagre:0.7.5 +- dagre-d3:0.4.17 +- ev-emitter:1.1.1 +- font-awesome:4.5.0 (CSS) +- graphlib:1.0.7 +- imagesloaded:4.1.4 +- jquery:2.2.0 +- lodash:3.10.1 +- moment:2.10.6 +- moment-duration-format:1.3.0 +- qtip2:2.2.1 +- Split.js:1.0.6 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- d3:3.5.12 + +This project bundles the following dependencies under SIL OFL 1.1 license. (https://opensource.org/licenses/OFL-1.1) +See bundled license files for details. + +- font-awesome:4.5.0 (Font) + +flink-swift-fs-hadoop +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-annotations:2.8.1 +- org.apache.hadoop:hadoop-auth:2.8.1 +- org.apache.hadoop:hadoop-client:2.8.1 +- org.apache.hadoop:hadoop-common:2.8.1 +- org.apache.hadoop:hadoop-hdfs:2.8.1 +- org.apache.hadoop:hadoop-hdfs-client:2.8.1 +- org.apache.hadoop:hadoop-openstack:2.8.1 +- org.apache.htrace:htrace-core4:4.0.1-incubating +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- commons-beanutils:commons-beanutils:1.8.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- commons-httpclient:commons-httpclient:3.1 +- com.google.guava:guava:11.0.2 +- com.google.code.gson:gson:2.2.4 +- com.squareup.okhttp:okhttp:2.4.0 +- com.squareup.okio:okio:1.4.0 +- net.minidev:json-smart:1.1.1 +- com.nimbusds:nimbus-jose-jwt:3.9 +- org.mortbay.jetty:jetty-sslengine:6.1.26 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +This project bundles the following dependencies under the BSD License. +See bundled license files for details. + +- xmlenc:xmlenc:0.52 +- com.google.protobuf:protobuf-java:2.5.0 + +This project bundles "net.jcip:jcip-annotations:1.0". +Written by Brian Goetz and Tim Peierls with assistance from members of JCP JSR-166 Expert Group +and released to the public domain, as explained by the Creative Commons public domain license. +https://creativecommons.org/licenses/publicdomain/ + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles org.apache.hadoop:*:2.8.1 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +/** + * Copyright 2008,2009,2010 Massachusetts Institute of Technology. + * All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +And the binary distribution of this product bundles these dependencies under the +following license: +Mockito 1.8.5 +SLF4J 1.7.10 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +-------------------------------------------------------------------------------- + +Copyright jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js.gz +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.0.0 +-------------------------------------------------------------------------------- +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +servlet-api 2.5 +jsp-api 2.1 +Streaming API for XML 1.0 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +Jersey 1.9 +JAXB API bundle for GlassFish V3 2.2.2 +JAXB RI 2.2.3 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +ecj-4.3.1.jar +-------------------------------------------------------------------------------- +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +ASM Core 3.2 +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +xmlenc Library 0.52 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +Java Concurrency in Practice book annotations 1.0 +-------------------------------------------------------------------------------- +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +flink-hadoop-fs +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom +* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni), +which has the following notices: +* This product includes software developed by FuseSource Corp. + http://fusesource.com +* This product includes software developed at + Progress Software Corporation and/or its subsidiaries or affiliates. +* This product includes software developed by IBM Corporation and others. + +The binary distribution of this product bundles binaries of +AWS Java SDK 1.10.6, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: + * license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: + * license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: + * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/libdivsufsort/ + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/snappy/ + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * http://code.google.com/p/caliper/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + +The binary distribution of this product bundles binaries of +Commons Codec 1.4, +which has the following notices: + * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + =============================================================================== + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +The binary distribution of this product bundles binaries of +Commons Lang 2.6, +which has the following notices: + * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +The binary distribution of this product bundles binaries of +Apache Log4j 1.2.17, +which has the following notices: + * ResolverUtil.java + Copyright 2005-2006 Tim Fennell + Dumbster SMTP test server + Copyright 2004 Jason Paul Kitchen + TypeUtil.java + Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +The binary distribution of this product bundles binaries of +Java Concurrency in Practice book annotations 1.0, +which has the following notices: + * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative + Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net Any republication or derived work + distributed in source code form must include this copyright and license + notice. + +The binary distribution of this product bundles binaries of +Jetty 6.1.26, +which has the following notices: + * ============================================================== + Jetty Web Container + Copyright 1995-2016 Mort Bay Consulting Pty Ltd. + ============================================================== + + The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd + unless otherwise noted. + + Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + + Jetty may be distributed under either license. + + ------ + Eclipse + + The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + + The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + ------ + Oracle + + The following artifacts are CDDL + GPLv2 with classpath exception. + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + + ------ + Oracle OpenJDK + + If ALPN is used to negotiate HTTP/2 connections, then the following + artifacts may be included in the distribution or downloaded when ALPN + module is selected. + + * java.sun.security.ssl + + These artifacts replace/modify OpenJDK classes. The modififications + are hosted at github and both modified and original are under GPL v2 with + classpath exceptions. + http://openjdk.java.net/legal/gplv2+ce.html + + ------ + OW2 + + The following artifacts are licensed by the OW2 Foundation according to the + terms of http://asm.ow2.org/license.html + + org.ow2.asm:asm-commons + org.ow2.asm:asm + + ------ + Apache + + The following artifacts are ASL2 licensed. + + org.apache.taglibs:taglibs-standard-spec + org.apache.taglibs:taglibs-standard-impl + + ------ + MortBay + + The following artifacts are ASL2 licensed. Based on selected classes from + following Apache Tomcat jars, all ASL2 licensed. + + org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + + org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + ------ + Mortbay + + The following artifacts are CDDL + GPLv2 with classpath exception. + + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + org.eclipse.jetty.toolchain:jetty-schemas + + ------ + Assorted + + The UnixCrypt.java code implements the one way cryptography used by + Unix systems for simple password protection. Copyright 1996 Aki Yoshida, + modified April 2001 by Iris Van den Broeke, Daniel Deville. + Permission to use, copy, modify and distribute UnixCrypt + for non-commercial or commercial purposes and without fee is + granted provided that the copyright notice appears in all copies./ + +The binary distribution of this product bundles binaries of +Snappy for Java 1.0.4.1, +which has the following notices: + * This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + == Contributors == + * Tatu Saloranta + * Providing benchmark suite + * Alec Wysoker + * Performance and memory usage improvement + +The binary distribution of this product bundles binaries of +Xerces2 Java Parser 2.9.1, +which has the following notices: + * ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache Xerces Java distribution. == + ========================================================================= + + Apache Xerces Java + Copyright 1999-2007 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. + - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. + - voluntary contributions made by Paul Eng on behalf of the + Apache Software Foundation that were originally developed at iClick, Inc., + software copyright (c) 1999. + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Net +Copyright 2001-2012 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Digester +Copyright 2001-2008 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2010 The Apache Software Foundation + +htrace-core4 +Copyright 2015 The Apache Software Foundation + +This product currently only contains code developed by authors +of specific components, as identified by the source code files; +if such notes are missing files have been created by +Tatu Saloranta. + +For additional credits (generally to people who reported problems) +see CREDITS file. + +Apache Jakarta HttpClient +Copyright 1999-2007 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +flink-shaded-hadoop2-uber +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.jamesmurty.utils:java-xmlbuilder:0.4 +- commons-beanutils:commons-beanutils:1.9.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-daemon:commons-daemon:1.0.13 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- org.apache.avro:avro:1.8.2 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- org.apache.zookeeper:zookeeper:3.4.10 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.jcraft:jsch:0.1.42 +- com.thoughtworks.paranamer:paranamer:2.7 +- xmlenc:xmlenc:0.52 + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.0. +See bundled license files for details. + +- javax.activation:activation:1.1 (http://java.sun.com/products/javabeans/jaf/index.jsp) +- javax.servlet:servlet-api:2.5 (https://repo1.maven.org/maven2/javax/servlet/servlet-api/2.5/) +- javax.xml.stream:stax-api:1.0-2 (http://central.maven.org/maven2/javax/xml/stream/stax-api/1.0-2/) + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.1. +See bundled license files for details. + +- com.sun.jersey:jersey-client:1.9 (http://central.maven.org/maven2/com/sun/jersey/jersey-client/1.9/) +- javax.xml.bind:jaxb-api:2.2.2 (https://jaxb.dev.java.net/) + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + + +flink-shaded-hadoop2 +Copyright 2014-2018 The Apache Software Foundation + +- com.google.guava:guava:11.0.2 +- net.java.dev.jets3t:jets3t:0.9.0 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- commons-httpclient:commons-httpclient:3.1 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +The version of the actual Hadoop dependencies depend on the built. + +- org.apache.hadoop:hadoop-common +- org.apache.hadoop:hadoop-hdfs +- org.apache.hadoop:hadoop-mapreduce-client-core +- org.apache.hadoop:hadoop-yarn-client +- org.apache.hadoop:hadoop-yarn-common +- org.apache.hadoop:hadoop-annotations +- org.apache.hadoop:hadoop-auth +- org.apache.hadoop:hadoop-yarn-api + +- com.google.protobuf.protobuf-java:2.5.0 + +This product currently only contains code developed by authors +of specific components, as identified by the source code files; +if such notes are missing files have been created by +Tatu Saloranta. + +For additional credits (generally to people who reported problems) +see CREDITS file. + +Apache Jakarta HttpClient +Copyright 1999-2007 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +Apache Avro +Copyright 2009-2017 The Apache Software Foundation + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Net +Copyright 2001-2012 The Apache Software Foundation + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Digester +Copyright 2001-2008 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2016 The Apache Software Foundation + +Apache Commons Daemon +Copyright 1999-2013 The Apache Software Foundation + +flink-s3-fs-presto +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.facebook.presto:presto-hive:0.185 +- com.facebook.presto.hadoop:hadoop-apache2:2.7.3-1 +- com.google.guava:guava:21.0 +- io.airlift:configuration:0.148 +- io.airlift:log:0.148 +- io.airlift:stats:0.148 +- io.airlift:units:1.0 +- io.airlift:slice:0.31 +- com.fasterxml.jackson.core:jackson-annotations:2.8.1 +- com.fasterxml.jackson.core:jackson-core:2.8.1 +- com.fasterxml.jackson.core:jackson-databind:2.8.1 +- joda-time:joda-time:2.5 +- org.weakref:jmxutils:1.19 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.9 + + +flink-s3-fs-base +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-aws:3.1.0 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpclient:4.5.3 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- com.amazonaws:aws-java-sdk-core:1.11.271 +- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 +- com.amazonaws:aws-java-sdk-kms:1.11.271 +- com.amazonaws:aws-java-sdk-s3:1.11.271 +- com.amazonaws:jmespath-java:1.11.271 +- software.amazon.ion:ion-java:1.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.6.0 +- com.fasterxml.jackson.core:jackson-core:2.6.7 +- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 +- joda-time:joda-time:2.5 + +flink-fs-hadoop-shaded +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-annotations:3.1.0 +- org.apache.hadoop:hadoop-auth:3.1.0 +- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.3.2 +- commons-lang:commons-lang:2.6 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.3 +- com.google.guava:guava:11.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.7.0 +- com.fasterxml.jackson.core:jackson-core:2.7.8 +- com.fasterxml.jackson.core:jackson-databind:2.7.8 +- com.fasterxml.woodstox:woodstox-core:5.0.3 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +See bundled license files for details. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +See bundled license files for details. + +- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) + +This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of the Massachusetts Institute of Technology nor + the names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Other portions are under the same license from Intel: +http://sourceforge.net/projects/slicing-by-8/ +/*++ + * + * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + * + * This software program is licensed subject to the BSD License, + * available at http://www.opensource.org/licenses/bsd-license.html + * + * Abstract: The main routine + * + --*/ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +For hadoop-common-project/hadoop-common/src/main/native/gtest +--------------------------------------------------------------------- +Copyright 2008, Google Inc. +All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles these dependencies under the +following license: +re2j 1.1 +--------------------------------------------------------------------- +(GO license) +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +The binary distribution of this product bundles these dependencies under the +following license: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +bootstrap v3.3.6 +broccoli-asset-rev v2.4.2 +broccoli-funnel v1.0.1 +datatables v1.10.8 +em-helpers v0.5.13 +em-table v0.1.6 +ember v2.2.0 +ember-array-contains-helper v1.0.2 +ember-bootstrap v0.5.1 +ember-cli v1.13.13 +ember-cli-app-version v1.0.0 +ember-cli-babel v5.1.6 +ember-cli-content-security-policy v0.4.0 +ember-cli-dependency-checker v1.2.0 +ember-cli-htmlbars v1.0.2 +ember-cli-htmlbars-inline-precompile v0.3.1 +ember-cli-ic-ajax v0.2.1 +ember-cli-inject-live-reload v1.4.0 +ember-cli-jquery-ui v0.0.20 +ember-cli-qunit v1.2.1 +ember-cli-release v0.2.8 +ember-cli-shims v0.0.6 +ember-cli-sri v1.2.1 +ember-cli-test-loader v0.2.1 +ember-cli-uglify v1.2.0 +ember-d3 v0.1.0 +ember-data v2.1.0 +ember-disable-proxy-controllers v1.0.1 +ember-export-application-global v1.0.5 +ember-load-initializers v0.1.7 +ember-qunit v0.4.16 +ember-qunit-notifications v0.1.0 +ember-resolver v2.0.3 +ember-spin-spinner v0.2.3 +ember-truth-helpers v1.2.0 +jquery v2.1.4 +jquery-ui v1.11.4 +loader.js v3.3.0 +momentjs v2.10.6 +qunit v1.19.0 +select2 v4.0.0 +snippet-ss v1.11.0 +spin.js v2.3.2 +Azure Data Lake Store - Java client SDK 2.0.11 +JCodings 1.0.8 +Joni 2.1.2 +Mockito 1.8.5 +JUL to SLF4J bridge 1.7.25 +SLF4J API Module 1.7.25 +SLF4J LOG4J-12 Binding 1.7.25 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 +Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2003-2017 Optimatika + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For: +oj! Algorithms - version 43.0 +-------------------------------------------------------------------------------- + +Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.3.4 +-------------------------------------------------------------------------------- +(HSQL License) +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +Java Servlet API 3.1.0 +servlet-api 2.5 +jsp-api 2.1 +jsr311-api 1.1.1 +Glassfish Jasper 6.1.14 +Servlet Specification 2.5 API 6.1.14 +-------------------------------------------------------------------------------- +(CDDL 1.0) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +jersey-client 1.19 +jersey-core 1.19 +jersey-grizzly2 1.19 +jersey-grizzly2-servlet 1.19 +jersey-json 1.19 +jersey-server 1.19 +jersey-servlet 1.19 +jersey-guice 1.19 +Jersey Test Framework - Grizzly 2 Module 1.19 +JAXB RI 2.2.3 +Java Architecture for XML Binding 2.2.11 +grizzly-framework 2.2.21 +grizzly-http 2.2.21 +grizzly-http-server 2.2.21 +grizzly-http-servlet 2.2.21 +grizzly-rcm 2.2.21 +-------------------------------------------------------------------------------- +(CDDL 1.1) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +Eclipse JDT Core 3.1.1 +-------------------------------------------------------------------------------- +(EPL v1.0) +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +ASM Core 5.0.4 +ASM Commons 5.0.2 +ASM Tree 5.0.2 +-------------------------------------------------------------------------------- +(3-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. +-------------------------------------------------------------------------------- +(2-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +"Java Concurrency in Practice" book annotations 1.0 +-------------------------------------------------------------------------------- +(CCAL v2.5) +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +The binary distribution of this product bundles these dependencies under the +following license: +jamon-runtime 2.4.1 +-------------------------------------------------------------------------------- +(MPL 2.0) + Mozilla Public License + Version 2.0 + +1.1. “Contributor” +means each individual or legal entity that creates, contributes to the creation +of, or owns Covered Software. + +1.2. “Contributor Version” +means the combination of the Contributions of others (if any) used by a +Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” +means Covered Software of a particular Contributor. + +1.4. “Covered Software” +means Source Code Form to which the initial Contributor has attached the notice +in Exhibit A, the Executable Form of such Source Code Form, and Modifications of +such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” +means + +that the initial Contributor has attached the notice described in Exhibit B to +the Covered Software; or + +that the Covered Software was made available under the terms of version 1.1 or +earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” +means any form of the work other than Source Code Form. + +1.7. “Larger Work” +means a work that combines Covered Software with other material, in a separate +file or files, that is not Covered Software. + +1.8. “License” +means this document. + +1.9. “Licensable” +means having the right to grant, to the maximum extent possible, whether at the +time of the initial grant or subsequently, any and all of the rights conveyed by +this License. + +1.10. “Modifications” +means any of the following: + +any file in Source Code Form that results from an addition to, deletion from, or +modification of the contents of Covered Software; or + +any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor +means any patent claim(s), including without limitation, method, process, and +apparatus claims, in any patent Licensable by such Contributor that would be +infringed, but for the grant of the License, by the making, using, selling, +offering for sale, having made, import, or transfer of either its Contributions +or its Contributor Version. + +1.12. “Secondary License” +means either the GNU General Public License, Version 2.0, the GNU Lesser General +Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, +or any later versions of those licenses. + +1.13. “Source Code Form” +means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) +means an individual or a legal entity exercising rights under this License. For +legal entities, “You” includes any entity that controls, is controlled by, +or is under common control with You. For purposes of this definition, +“control” means (a) the power, direct or indirect, to cause the direction or +management of such entity, whether by contract or otherwise, or (b) ownership of +more than fifty percent (50%) of the outstanding shares or beneficial ownership +of such entity. + +2. License Grants and Conditions + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license: + +under intellectual property rights (other than patent or trademark) Licensable +by such Contributor to use, reproduce, make available, modify, display, perform, +distribute, and otherwise exploit its Contributions, either on an unmodified +basis, with Modifications, or as part of a Larger Work; and + +under Patent Claims of such Contributor to make, use, sell, offer for sale, have +made, import, and otherwise transfer either its Contributions or its Contributor +Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution become +effective for each Contribution on the date the Contributor first distributes +such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under this +License. No additional rights or licenses will be implied from the distribution +or licensing of Covered Software under this License. Notwithstanding Section +2.1(b) above, no patent license is granted by a Contributor: + +for any code that a Contributor has removed from Covered Software; or + +for infringements caused by: (i) Your and any other third party’s +modifications of Covered Software, or (ii) the combination of its Contributions +with other software (except as part of its Contributor Version); or + +under Patent Claims infringed by Covered Software in the absence of its +Contributions. + +This License does not grant any rights in the trademarks, service marks, or +logos of any Contributor (except as may be necessary to comply with the notice +requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to distribute +the Covered Software under a subsequent version of this License (see Section +10.2) or under the terms of a Secondary License (if permitted under the terms of +Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its Contributions are +its original creation(s) or it has sufficient rights to grant the rights to its +Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under applicable +copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in +Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under the +terms of this License. You must inform recipients that the Source Code Form of +the Covered Software is governed by the terms of this License, and how they can +obtain a copy of this License. You may not attempt to alter or restrict the +recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +such Covered Software must also be made available in Source Code Form, as +described in Section 3.1, and You must inform recipients of the Executable Form +how they can obtain a copy of such Source Code Form by reasonable means in a +timely manner, at a charge no more than the cost of distribution to the +recipient; and + +You may distribute such Executable Form under the terms of this License, or +sublicense it under different terms, provided that the license for the +Executable Form does not attempt to limit or alter the recipients’ rights in +the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, provided +that You also comply with the requirements of this License for the Covered +Software. If the Larger Work is a combination of Covered Software with a work +governed by one or more Secondary Licenses, and the Covered Software is not +Incompatible With Secondary Licenses, this License permits You to additionally +distribute such Covered Software under the terms of such Secondary License(s), +so that the recipient of the Larger Work may, at their option, further +distribute the Covered Software under the terms of either this License or such +Secondary License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices (including +copyright notices, patent notices, disclaimers of warranty, or limitations of +liability) contained within the Source Code Form of the Covered Software, except +that You may alter any license notices to the extent required to remedy known +factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, indemnity +or liability obligations to one or more recipients of Covered Software. However, +You may do so only on Your own behalf, and not on behalf of any Contributor. You +must make it absolutely clear that any such warranty, support, indemnity, or +liability obligation is offered by You alone, and You hereby agree to indemnify +every Contributor for any liability incurred by such Contributor as a result of +warranty, support, indemnity or liability terms You offer. You may include +additional disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with +respect to some or all of the Covered Software due to statute, judicial order, +or regulation then You must: (a) comply with the terms of this License to the +maximum extent possible; and (b) describe the limitations and the code they +affect. Such description must be placed in a text file included with all +distributions of the Covered Software under this License. Except to the extent +prohibited by statute or regulation, such description must be sufficiently +detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You +fail to comply with any of its terms. However, if You become compliant, then the +rights granted under this License from a particular Contributor are reinstated +(a) provisionally, unless and until such Contributor explicitly and finally +terminates Your grants, and (b) on an ongoing basis, if such Contributor fails +to notify You of the non-compliance by some reasonable means prior to 60 days +after You have come back into compliance. Moreover, Your grants from a +particular Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the first +time You have received notice of non-compliance with this License from such +Contributor, and You become compliant prior to 30 days after Your receipt of the +notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, counter-claims, and +cross-claims) alleging that a Contributor Version directly or indirectly +infringes any patent, then the rights granted to You by any and all Contributors +for the Covered Software under Section 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user +license agreements (excluding distributors and resellers) which have been +validly granted by You or Your distributors under this License prior to +termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without +warranty of any kind, either expressed, implied, or statutory, including, +without limitation, warranties that the Covered Software is free of defects, +merchantable, fit for a particular purpose or non-infringing. The entire risk as +to the quality and performance of the Covered Software is with You. Should any +Covered Software prove defective in any respect, You (not any Contributor) +assume the cost of any necessary servicing, repair, or correction. This +disclaimer of warranty constitutes an essential part of this License. No use of +any Covered Software is authorized under this License except under this +disclaimer. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort (including +negligence), contract, or otherwise, shall any Contributor, or anyone who +distributes Covered Software as permitted above, be liable to You for any +direct, indirect, special, incidental, or consequential damages of any character +including, without limitation, damages for lost profits, loss of goodwill, work +stoppage, computer failure or malfunction, or any and all other commercial +damages or losses, even if such party shall have been informed of the +possibility of such damages. This limitation of liability shall not apply to +liability for death or personal injury resulting from such party’s negligence +to the extent applicable law prohibits such limitation. Some jurisdictions do +not allow the exclusion or limitation of incidental or consequential damages, so +this exclusion and limitation may not apply to You. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a +jurisdiction where the defendant maintains its principal place of business and +such litigation shall be governed by laws of that jurisdiction, without +reference to its conflict-of-law provisions. Nothing in this Section shall +prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter +hereof. If any provision of this License is held to be unenforceable, such +provision shall be reformed only to the extent necessary to make it enforceable. +Any law or regulation which provides that the language of a contract shall be +construed against the drafter shall not be used to construe this License against +a Contributor. + +10. Versions of the License + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section 10.3, +no one other than the license steward has the right to modify or publish new +versions of this License. Each version will be given a distinguishing version +number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version of the +License under which You originally received the Covered Software, or under the +terms of any subsequent version published by the license steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to create a +new license for such software, you may create and use a modified version of this +License if you rename the license and remove any references to the name of the +license steward (except to note that such modified license differs from this +License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + +If You choose to distribute Source Code Form that is Incompatible With Secondary +Licenses under the terms of this version of the License, the notice described in +Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not distributed with this file, You can obtain one +at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined +by the Mozilla Public License, v. 2.0. + +The binary distribution of this product bundles these dependencies under the +following license: +JDOM 1.1 +-------------------------------------------------------------------------------- +/*-- + + Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the disclaimer that follows + these conditions in the documentation and/or other materials + provided with the distribution. + + 3. The name "JDOM" must not be used to endorse or promote products + derived from this software without prior written permission. For + written permission, please contact . + + 4. Products derived from this software may not be called "JDOM", nor + may "JDOM" appear in their name, without prior written permission + from the JDOM Project Management . + + In addition, we request (but do not require) that you include in the + end-user documentation provided with the redistribution and/or in the + software itself an acknowledgement equivalent to the following: + "This product includes software developed by the + JDOM Project (http://www.jdom.org/)." + Alternatively, the acknowledgment may be graphical using the logos + available at http://www.jdom.org/images/logos. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + + This software consists of voluntary contributions made by many + individuals on behalf of the JDOM Project and was originally + created by Jason Hunter and + Brett McLaughlin . For more information + on the JDOM Project, please see . + + */ + +The binary distribution of this product bundles these dependencies under the +following license: +Hbase Server 1.2.4 +-------------------------------------------------------------------------------- +This project bundles a derivative image for our Orca Logo. This image is +available under the Creative Commons By Attribution 3.0 License. + + Creative Commons Legal Code + + Attribution 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + + License + + THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE + COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY + COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS + AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + + BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE + TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY + BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS + CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND + CONDITIONS. + + 1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined above) for the purposes of this + License. + c. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + d. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + e. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + f. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + g. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + h. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + i. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + + 2. Fair Dealing Rights. Nothing in this License is intended to reduce, + limit, or restrict any uses free from copyright or rights arising from + limitations or exceptions that are provided for in connection with the + copyright protection under copyright law or other applicable laws. + + 3. License Grant. Subject to the terms and conditions of this License, + Licensor hereby grants You a worldwide, royalty-free, non-exclusive, + perpetual (for the duration of the applicable copyright) license to + exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + + The above rights may be exercised in all media and formats whether now + known or hereafter devised. The above rights include the right to make + such modifications as are technically necessary to exercise the rights in + other media and formats. Subject to Section 8(f), all rights not expressly + granted by Licensor are hereby reserved. + + 4. Restrictions. The license granted in Section 3 above is expressly made + subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(b), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(b), as requested. + b. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Section 3(b), in the case of an Adaptation, + a credit identifying the use of the Work in the Adaptation (e.g., + "French translation of the Work by Original Author," or "Screenplay + based on original Work by Original Author"). The credit required by + this Section 4 (b) may be implemented in any reasonable manner; + provided, however, that in the case of a Adaptation or Collection, at + a minimum such credit will appear, if a credit for all contributing + authors of the Adaptation or Collection appears, then as part of these + credits and in a manner at least as prominent as the credits for the + other contributing authors. For the avoidance of doubt, You may only + use the credit required by this Section for the purpose of attribution + in the manner set out above and, by exercising Your rights under this + License, You may not implicitly or explicitly assert or imply any + connection with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of You or Your + use of the Work, without the separate, express prior written + permission of the Original Author, Licensor and/or Attribution + Parties. + c. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + + 5. Representations, Warranties and Disclaimer + + UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR + OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY + KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, + INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, + FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF + LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, + WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION + OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + + 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE + LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR + ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES + ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS + BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + + 8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of this License. + + Creative Commons may be contacted at https://creativecommons.org/. +-------------------------------------------------------------------------------- + +For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/AbstractFuture.java and +hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/TimeoutFuture.java + +Copyright (C) 2007 The Guava Authors + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom +* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +AWS SDK for Java - Bundle 1.11.134, +AWS Java SDK for AWS KMS 1.11.134, +AWS Java SDK for Amazon S3 1.11.134, +AWS Java SDK for AWS STS 1.11.134, +JMES Path Query library 1.0, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: + * license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: + * license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: + * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/libdivsufsort/ + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/snappy/ + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * http://code.google.com/p/caliper/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + +This product contains a modified portion of 'JDOM 1.1', which can be obtained at: + + * LICENSE: + * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt + * HOMEPAGE: + * http://www.jdom.org/ + +The binary distribution of this product bundles binaries of +Commons Codec 1.4, +which has the following notices: + * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + =============================================================================== + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +The binary distribution of this product bundles binaries of +Commons Lang 2.6, +which has the following notices: + * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +The binary distribution of this product bundles binaries of +Apache Log4j 1.2.17, +which has the following notices: + * ResolverUtil.java + Copyright 2005-2006 Tim Fennell + Dumbster SMTP test server + Copyright 2004 Jason Paul Kitchen + TypeUtil.java + Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +The binary distribution of this product bundles binaries of +"Java Concurrency in Practice" book annotations 1.0, +which has the following notices: + * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative + Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net Any republication or derived work + distributed in source code form must include this copyright and license + notice. + +The binary distribution of this product bundles binaries of +Jetty :: Http Utility 9.3.19., +Jetty :: IO Utility 9.3.19., +Jetty :: Security 9.3.19., +Jetty :: Server Core 9.3.19., +Jetty :: Servlet Handling 9.3.19., +Jetty :: Utilities 9.3.19., +Jetty :: Utilities :: Ajax, +Jetty :: Webapp Application Support 9.3.19., +Jetty :: XML utilities 9.3.19., +which has the following notices: + * ============================================================== + Jetty Web Container + Copyright 1995-2016 Mort Bay Consulting Pty Ltd. + ============================================================== + + The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd + unless otherwise noted. + + Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + + Jetty may be distributed under either license. + + ------ + Eclipse + + The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + + The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + ------ + Oracle + + The following artifacts are CDDL + GPLv2 with classpath exception. + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + + ------ + Oracle OpenJDK + + If ALPN is used to negotiate HTTP/2 connections, then the following + artifacts may be included in the distribution or downloaded when ALPN + module is selected. + + * java.sun.security.ssl + + These artifacts replace/modify OpenJDK classes. The modififications + are hosted at github and both modified and original are under GPL v2 with + classpath exceptions. + http://openjdk.java.net/legal/gplv2+ce.html + + ------ + OW2 + + The following artifacts are licensed by the OW2 Foundation according to the + terms of http://asm.ow2.org/license.html + + org.ow2.asm:asm-commons + org.ow2.asm:asm + + ------ + Apache + + The following artifacts are ASL2 licensed. + + org.apache.taglibs:taglibs-standard-spec + org.apache.taglibs:taglibs-standard-impl + + ------ + MortBay + + The following artifacts are ASL2 licensed. Based on selected classes from + following Apache Tomcat jars, all ASL2 licensed. + + org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + + org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + ------ + Mortbay + + The following artifacts are CDDL + GPLv2 with classpath exception. + + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + org.eclipse.jetty.toolchain:jetty-schemas + + ------ + Assorted + + The UnixCrypt.java code implements the one way cryptography used by + Unix systems for simple password protection. Copyright 1996 Aki Yoshida, + modified April 2001 by Iris Van den Broeke, Daniel Deville. + Permission to use, copy, modify and distribute UnixCrypt + for non-commercial or commercial purposes and without fee is + granted provided that the copyright notice appears in all copies./ + +The binary distribution of this product bundles binaries of +Snappy for Java 1.0.4.1, +which has the following notices: + * This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + == Contributors == + * Tatu Saloranta + * Providing benchmark suite + * Alec Wysoker + * Performance and memory usage improvement + +The binary distribution of this product bundles binaries of +Xerces2 Java Parser 2.9.1, +which has the following notices: + * ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache Xerces Java distribution. == + ========================================================================= + + Apache Xerces Java + Copyright 1999-2007 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. + - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. + - voluntary contributions made by Paul Eng on behalf of the + Apache Software Foundation that were originally developed at iClick, Inc., + software copyright (c) 1999. + +The binary distribution of this product bundles binaries of +Logback Classic Module 1.1.2, +Logback Core Module 1.1.2, +which has the following notices: + * Logback: the reliable, generic, fast and flexible logging framework. + Copyright (C) 1999-2012, QOS.ch. All rights reserved. + +The binary distribution of this product bundles binaries of +Apache HBase - Annotations 1.2.6, +Apache HBase - Client 1.2.6, +Apache HBase - Common 1.2.6, +Apache HBase - Hadoop Compatibility 1.2.6, +Apache HBase - Hadoop Two Compatibility 1.2.6, +Apache HBase - Prefix Tree 1.2.6, +Apache HBase - Procedure 1.2.6, +Apache HBase - Protocol 1.2.6, +Apache HBase - Server 1.2.6, +which has the following notices: + * Apache HBase + Copyright 2007-2015 The Apache Software Foundation + + -- + This product incorporates portions of the 'Hadoop' project + + Copyright 2007-2009 The Apache Software Foundation + + Licensed under the Apache License v2.0 + -- + Our Orca logo we got here: http://www.vectorfree.com/jumping-orca + It is licensed Creative Commons Attribution 3.0. + See https://creativecommons.org/licenses/by/3.0/us/ + We changed the logo by stripping the colored background, inverting + it and then rotating it some. + + Later we found that vectorfree.com image is not properly licensed. + The original is owned by vectorportal.com. The original was + relicensed so we could use it as Creative Commons Attribution 3.0. + The license is bundled with the download available here: + http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp + -- + This product includes portions of the Bootstrap project v3.0.0 + + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License v2.0 + + This product uses the Glyphicons Halflings icon set. + + http://glyphicons.com/ + + Copyright Jan Kovařík + + Licensed under the Apache License v2.0 as a part of the Bootstrap project. + + -- + This product includes portions of the Guava project v14, specifically + 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' + + Copyright (C) 2007 The Guava Authors + + Licensed under the Apache License, Version 2.0 + +The binary distribution of this product bundles binaries of +Phoenix Core 4.7.0, +which has the following notices: + Apache Phoenix + Copyright 2013-2016 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This also includes: + + The phoenix-spark module has been adapted from the phoenix-spark library + distributed under the terms of the Apache 2 license. Original source copyright: + Copyright 2014 Simply Measured, Inc. + Copyright 2015 Interset Software Inc. + + The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 + (https://pypi.python.org/pypi/python-daemon/). Original source copyright: + # Copyright © 2008–2015 Ben Finney + # Copyright © 2007–2008 Robert Niederreiter, Jens Klein + # Copyright © 2004–2005 Chad J. Schroeder + # Copyright © 2003 Clark Evans + # Copyright © 2002 Noah Spurrier + # Copyright © 2001 Jürgen Hermann + +The binary distribution of this product bundles binaries of +Plexus Cipher: encryption/decryption Component 1.4, +which has the following notices: + * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java + which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 + + The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java + which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 + +The binary distribution of this product bundles binaries of +software.amazon.ion:ion-java 1.0.1, +which has the following notices: + * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +The binary distribution of this product bundles binaries of +joda-time:joda-time:2.9.9 +which has the following notices: + * ============================================================================= + = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = + ============================================================================= + This product includes software developed by + Joda.org (http://www.joda.org/). + +The binary distribution of this product bundles binaries of +Ehcache 3.3.1, +which has the following notices: + * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. + +The binary distribution of this product bundles binaries of +snakeyaml (https://bitbucket.org/asomov/snakeyaml), +which has the following notices: + * Copyright (c) 2008, http://www.snakeyaml.org + +The binary distribution of this product bundles binaries of +swagger-annotations (https://github.com/swagger-api/swagger-core), +which has the following notices: + * Copyright 2016 SmartBear Software + +The binary distribution of this product bundles binaries of +metrics-core 3.2.4 +which has the following notices: + * Copyright 2010-2013 Coda Hale and Yammer, Inc. + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, + LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2016 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2017 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2014 The Apache Software Foundation + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +htrace-core4 +Copyright 2016 The Apache Software Foundation + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +force-shading +Copyright 2018 The Apache Software Foundation + +flink-hadoop-fs +Copyright 2014-2018 The Apache Software Foundation + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +flink-metrics-prometheus +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.prometheus:simpleclient:0.3.0 +- io.prometheus:simpleclient_common:0.3.0 +- io.prometheus:simpleclient_httpserver:0.3.0 +- io.prometheus:simpleclient_pushgateway:0.3.0 + +flink-s3-fs-base +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-aws:3.1.0 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpclient:4.5.3 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- com.amazonaws:aws-java-sdk-core:1.11.271 +- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 +- com.amazonaws:aws-java-sdk-kms:1.11.271 +- com.amazonaws:aws-java-sdk-s3:1.11.271 +- com.amazonaws:jmespath-java:1.11.271 +- software.amazon.ion:ion-java:1.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.6.0 +- com.fasterxml.jackson.core:jackson-core:2.6.7 +- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 +- joda-time:joda-time:2.5 + +flink-fs-hadoop-shaded +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-annotations:3.1.0 +- org.apache.hadoop:hadoop-auth:3.1.0 +- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.3.2 +- commons-lang:commons-lang:2.6 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.3 +- com.google.guava:guava:11.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.7.0 +- com.fasterxml.jackson.core:jackson-core:2.7.8 +- com.fasterxml.jackson.core:jackson-databind:2.7.8 +- com.fasterxml.woodstox:woodstox-core:5.0.3 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +See bundled license files for details. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +See bundled license files for details. + +- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) + +flink-sql-client +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- org.jline:jline-terminal:3.9.0 +- org.jline:jline-reader:3.9.0 + + +flink-table +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.esri.geometry:esri-geometry-api:2.0.0 +- com.google.guava:guava:19.0 +- joda-time:joda-time:2.5 +- net.hydromatic:aggdesigner-algorithm:6.0 +- org.apache.calcite:calcite-core:1.17.0 +- org.apache.calcite:calcite-linq4j:1.17.0 +- org.apache.calcite.avatica:avatica-core:1.12.0 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details + +- org.codehaus.janino:janino:3.0.7 +- org.codehaus.janino:commons-compiler:3.0.7 + +flink-table-common +Copyright 2014-2018 The Apache Software Foundation + +Calcite Core +Copyright 2012-2018 The Apache Software Foundation + +Apache Calcite Avatica +Copyright 2012-2018 The Apache Software Foundation + +Calcite Linq4j +Copyright 2012-2018 The Apache Software Foundation + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +force-shading +Copyright 2018 The Apache Software Foundation + +flink-cep +Copyright 2014-2018 The Apache Software Foundation + +flink-streaming-python +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php) +See bundled license files for details. + +- org.python:jython-standalone:2.7.1 + +flink-metrics-graphite +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.dropwizard.metrics:metrics-core:3.1.5 +- io.dropwizard.metrics:metrics-graphite:3.1.5 + +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +flink-examples-streaming-state-machine +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 + + +flink-connector-kafka-0.10 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-0.9 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-base +Copyright 2014-2019 The Apache Software Foundation + +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + +flink-examples-streaming-twitter +Copyright 2014-2019 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +flink-connector-twitter +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:14.0.1 +- com.twitter:hbc-core:2.2.0 +- com.twitter:joauth:6.0.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation \ No newline at end of file diff --git a/README.md b/README.md index 7966ecd3d7659..64868e1869ccc 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ The IntelliJ IDE supports Maven out of the box and offers a plugin for Scala dev * IntelliJ download: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/) * IntelliJ Scala Plugin: [http://plugins.jetbrains.com/plugin/?id=1347](http://plugins.jetbrains.com/plugin/?id=1347) -Check out our [Setting up IntelliJ](https://github.com/apache/flink/blob/master/docs/internals/ide_setup.md#intellij-idea) guide for details. +Check out our [Setting up IntelliJ](https://ci.apache.org/projects/flink/flink-docs-master/flinkDev/ide_setup.html#intellij-idea) guide for details. ### Eclipse Scala IDE diff --git a/docs/_config.yml b/docs/_config.yml index fac4aabccbb43..3819e27463d01 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,12 +27,12 @@ # we change the version for the complete docs when forking of a release branch # etc. # The full version string as referenced in Maven (e.g. 1.2.1) -version: "1.7-SNAPSHOT" +version: "1.7.2" # For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot # release this should be the same as the regular version -version_title: "1.7-SNAPSHOT" +version_title: "1.7" # Branch on Github for this version -github_branch: "master" +github_branch: "release-1.7" # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.11" @@ -45,13 +45,13 @@ github_url: "https://github.com/apache/flink" download_url: "http://flink.apache.org/downloads.html" # please use a protocol relative URL here -baseurl: //ci.apache.org/projects/flink/flink-docs-master +baseurl: //ci.apache.org/projects/flink/flink-docs-release-1.7 stable_baseurl: //ci.apache.org/projects/flink/flink-docs-stable -javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master +javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-release-1.7 # Flag whether this is a stable version or not. Used for the quickstart page. -is_stable: false +is_stable: true # Flag to indicate whether an outdated warning should be shown. show_outdated_warning: false diff --git a/docs/_includes/generated/checkpointing_configuration.html b/docs/_includes/generated/checkpointing_configuration.html index 8f5ce7b341d13..6ad5eeeb09108 100644 --- a/docs/_includes/generated/checkpointing_configuration.html +++ b/docs/_includes/generated/checkpointing_configuration.html @@ -30,7 +30,7 @@
state.backend.local-recovery
false - + This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.
state.checkpoints.dir
@@ -50,7 +50,7 @@
taskmanager.state.local.root-dirs
(none) - + The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option diff --git a/docs/_includes/generated/cluster_configuration.html b/docs/_includes/generated/cluster_configuration.html new file mode 100644 index 0000000000000..c45a6dd1bed84 --- /dev/null +++ b/docs/_includes/generated/cluster_configuration.html @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + +
KeyDefaultDescription
cluster.services.shutdown-timeout
30000The shutdown timeout for cluster services like executors in milliseconds.
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html index 0458af24c06eb..99eec1dcec7af 100644 --- a/docs/_includes/generated/job_manager_configuration.html +++ b/docs/_includes/generated/job_manager_configuration.html @@ -17,11 +17,6 @@ 16 The maximum number of prior execution attempts kept in history. - -
jobmanager.execution.failover-strategy
- "full" - This option specifies how the job computation recovers from task failures. Accepted values are:
  • 'full': Restarts all tasks.
  • 'individual': Restarts only the failed task. Should only be used if all tasks are independent components.
  • 'region': Restarts all tasks that could be affected by the task failure.
-
jobmanager.heap.size
"1024m" diff --git a/docs/_includes/generated/mesos_configuration.html b/docs/_includes/generated/mesos_configuration.html index 54e92e5680c51..9d5be9f358026 100644 --- a/docs/_includes/generated/mesos_configuration.html +++ b/docs/_includes/generated/mesos_configuration.html @@ -12,21 +12,11 @@ 604800 The failover timeout in seconds for the Mesos scheduler, after which running tasks are automatically shut down. - -
mesos.initial-tasks
- 0 - The initial workers to bring up when the master starts. This option is ignored unless Flink is in legacy mode. -
mesos.master
(none) The Mesos master URL. The value should be in one of the following forms:
  • host:port
  • zk://host1:port1,host2:port2,.../path
  • zk://username:password@host1:port1,host2:port2,.../path
  • file:///path/to/file
- -
mesos.maximum-failed-tasks
- -1 - The maximum number of failed workers before the cluster fails. May be set to -1 to disable this feature. This option is ignored unless Flink is in legacy mode. -
mesos.resourcemanager.artifactserver.port
0 diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html index 16c5d98f8be80..c11faa61139d3 100644 --- a/docs/_includes/generated/task_manager_configuration.html +++ b/docs/_includes/generated/task_manager_configuration.html @@ -67,31 +67,6 @@ false Whether to kill the TaskManager when the task thread throws an OutOfMemoryError. - -
taskmanager.memory.fraction
- 0.7 - The relative amount of memory (after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. This parameter is only evaluated, if taskmanager.memory.size is not set. - - -
taskmanager.memory.off-heap
- false - Memory allocation method (JVM heap or off-heap), used for managed memory of the TaskManager as well as the network buffers. - - -
taskmanager.memory.preallocate
- false - Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting. - - -
taskmanager.memory.segment-size
- "32kb" - Size of memory buffers used by the network stack and the memory manager. - - -
taskmanager.memory.size
- "0" - Amount of memory to be allocated by the task manager's memory manager. If not set, a relative fraction will be allocated. -
taskmanager.network.detailed-metrics
false @@ -125,12 +100,12 @@
taskmanager.network.request-backoff.initial
100 - Minimum backoff for partition requests of input channels. + Minimum backoff in milliseconds for partition requests of input channels.
taskmanager.network.request-backoff.max
10000 - Maximum backoff for partition requests of input channels. + Maximum backoff in milliseconds for partition requests of input channels.
taskmanager.numberOfTaskSlots
diff --git a/docs/_includes/generated/task_manager_memory_configuration.html b/docs/_includes/generated/task_manager_memory_configuration.html new file mode 100644 index 0000000000000..306a1fd378307 --- /dev/null +++ b/docs/_includes/generated/task_manager_memory_configuration.html @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
taskmanager.memory.fraction
0.7The relative amount of memory (after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on taskmanager.memory.off-heap) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. This parameter is only evaluated, if taskmanager.memory.size is not set.
taskmanager.memory.off-heap
falseMemory allocation method (JVM heap or off-heap), used for managed memory of the TaskManager. For setups with larger quantities of memory, this can improve the efficiency of the operations performed on the memory.
When set to true, then it is advised that `taskmanager.memory.preallocate` is also set to true.
taskmanager.memory.preallocate
falseWhether TaskManager managed memory should be pre-allocated when the TaskManager is starting. When `taskmanager.memory.off-heap` is set to true, then it is advised that this configuration is also set to true. If this configuration is set to false cleaning up of the allocated off-heap memory happens only when the configured JVM parameter MaxDirectMemorySize is reached by triggering a full GC. For streaming setups is is highly recommended to set this value to false as the core state backends currently do not use the managed memory.
taskmanager.memory.segment-size
"32kb"Size of memory buffers used by the network stack and the memory manager.
taskmanager.memory.size
"0"The amount of memory (in megabytes) that the task manager reserves on-heap or off-heap (depending on taskmanager.memory.off-heap) for sorting, hash tables, and caching of intermediate results. If unspecified, the memory manager will take a fixed ratio with respect to the size of the task manager JVM as specified by taskmanager.memory.fraction.
diff --git a/docs/dev/batch/dataset_transformations.md b/docs/dev/batch/dataset_transformations.md index 96c04c9d54847..cf2da3fa91a0c 100644 --- a/docs/dev/batch/dataset_transformations.md +++ b/docs/dev/batch/dataset_transformations.md @@ -712,7 +712,7 @@ class MyCombinableGroupReducer out: Collector[String]): Unit = { val r: (String, Int) = - in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) + in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) // concat key and sum and emit out.collect (r._1 + "-" + r._2) } @@ -722,7 +722,7 @@ class MyCombinableGroupReducer out: Collector[(String, Int)]): Unit = { val r: (String, Int) = - in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) + in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) // emit tuple with key and sum out.collect(r) } diff --git a/docs/dev/batch/index.md b/docs/dev/batch/index.md index d0043647227df..0a498dfe2a1f6 100644 --- a/docs/dev/batch/index.md +++ b/docs/dev/batch/index.md @@ -401,12 +401,14 @@ DataSet result = in.partitionByRange(0) Custom Partitioning -

Manually specify a partitioning over the data. +

Assigns records based on a key to a specific partition using a custom Partitioner function. + The key can be specified as position key, expression key, and key selector function.
- Note: This method works only on single field keys.

+ Note: This method only works with a single field key.

{% highlight java %} DataSet> in = // [...] -DataSet result = in.partitionCustom(Partitioner partitioner, key) +DataSet result = in.partitionCustom(partitioner, key) + .mapPartition(new PartitionMapper()); {% endhighlight %} @@ -703,13 +705,14 @@ val result = in.partitionByRange(0).mapPartition { ... } Custom Partitioning -

Manually specify a partitioning over the data. +

Assigns records based on a key to a specific partition using a custom Partitioner function. + The key can be specified as position key, expression key, and key selector function.
- Note: This method works only on single field keys.

+ Note: This method only works with a single field key.

{% highlight scala %} val in: DataSet[(Int, String)] = // [...] val result = in - .partitionCustom(partitioner: Partitioner[K], key) + .partitionCustom(partitioner, key).mapPartition { ... } {% endhighlight %} diff --git a/docs/dev/batch/python.md b/docs/dev/batch/python.md index 486aa18c9b768..d36d7b15c91ef 100644 --- a/docs/dev/batch/python.md +++ b/docs/dev/batch/python.md @@ -435,7 +435,7 @@ class MyDeserializer(object): return MyObj(i) -env.register_custom_type(MyObj, MySerializer(), MyDeserializer()) +env.register_type(MyObj, MySerializer(), MyDeserializer()) {% endhighlight %} #### Tuples/Lists diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md index f81080a4b5315..e769faa7a465f 100644 --- a/docs/dev/connectors/kafka.md +++ b/docs/dev/connectors/kafka.md @@ -73,7 +73,7 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is This connector supports Kafka messages with timestamps both for producing and consuming. - flink-connector-kafka-0.11_2.11 + flink-connector-kafka-0.11{{ site.scala_version_suffix }} 1.4.0 FlinkKafkaConsumer011
FlinkKafkaProducer011 @@ -81,13 +81,25 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is Since 0.11.x Kafka does not support scala 2.10. This connector supports Kafka transactional messaging to provide exactly once semantic for the producer. - flink-connector-kafka_2.11 + flink-connector-kafka{{ site.scala_version_suffix }} 1.7.0 FlinkKafkaConsumer
FlinkKafkaProducer >= 1.0.0 - This Kafka connector attempts to track the latest version of the Kafka client. The version of the client it uses may change between Flink releases. Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later. However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated flink-connector-kafka-0.11 and link-connector-kafka-0.10 respectively. - + + This universal Kafka connector attempts to track the latest version of the Kafka client. + The version of the client it uses may change between Flink releases. + Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later. + However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated + flink-connector-kafka-0.11{{ site.scala_version_suffix }} and link-connector-kafka-0.10{{ site.scala_version_suffix }} respectively. +
+ Attention: as of Flink 1.7 the universal Kafka connector is considered to be + in a BETA status and might not be as stable as the 0.11 connector. + In case of problems with the universal connector, you can try to use flink-connector-kafka-0.11{{ site.scala_version_suffix }} + which should be compatible with all of the Kafka versions starting from 0.11. +
+ + @@ -101,7 +113,8 @@ Then, import the connector in your maven project: {% endhighlight %} -Note that the streaming connectors are currently not part of the binary distribution. See how to link with them for cluster execution [here]({{ site.baseurl}}/dev/linking.html). +Note that the streaming connectors are currently not part of the binary distribution. +See how to link with them for cluster execution [here]({{ site.baseurl}}/dev/linking.html). ## Installing Apache Kafka @@ -110,17 +123,21 @@ Note that the streaming connectors are currently not part of the binary distribu ## Kafka 1.0.0+ Connector -Starting with Flink 1.7, there is a new Kafka connector that does not track a specific Kafka major version. Rather, it tracks the latest version of Kafka at the time of the Flink release. +Starting with Flink 1.7, there is a new universal Kafka connector that does not track a specific Kafka major version. +Rather, it tracks the latest version of Kafka at the time of the Flink release. -If your Kafka broker version is 1.0.0 or newer, you should use this Kafka connector. If you use an older version of Kafka (0.11, 0.10, 0.9, or 0.8), you should use the connector corresponding to the broker version. +If your Kafka broker version is 1.0.0 or newer, you should use this Kafka connector. +If you use an older version of Kafka (0.11, 0.10, 0.9, or 0.8), you should use the connector corresponding to the broker version. ### Compatibility -The modern Kafka connector is compatible with older and newer Kafka brokers through the compatibility guarantees of the Kafka client API and broker. The modern Kafka connector is compatible with broker versions 0.11.0 or later, depending on the features used. For details on Kafka compatibility, please refer to the [Kafka documentation](https://kafka.apache.org/protocol.html#protocol_compatibility). +The universal Kafka connector is compatible with older and newer Kafka brokers through the compatibility guarantees of the Kafka client API and broker. +It is compatible with broker versions 0.11.0 or newer, depending on the features used. +For details on Kafka compatibility, please refer to the [Kafka documentation](https://kafka.apache.org/protocol.html#protocol_compatibility). ### Usage -The use of the modern Kafka connector add a dependency to it: +To use the universal Kafka connector add a dependency to it: {% highlight xml %} @@ -130,11 +147,14 @@ The use of the modern Kafka connector add a dependency to it: {% endhighlight %} -Then instantiate the new source (`FlinkKafkaConsumer`) and sink (`FlinkKafkaProducer`). The API is the backwards compatible with the older Kafka connectors. +Then instantiate the new source (`FlinkKafkaConsumer`) and sink (`FlinkKafkaProducer`). +The API is backward compatible with the Kafka 0.11 connector, +except of dropping specific Kafka version from the module and class names. ## Kafka Consumer -Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or 09 for Kafka 0.9.0.x versions, etc. or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). It provides access to one or more Kafka topics. +Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or 09 for Kafka 0.9.0.x versions, etc. +or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). It provides access to one or more Kafka topics. The constructor accepts the following arguments: @@ -516,7 +536,10 @@ the `Watermark getCurrentWatermark()` (for periodic) or the `Watermark checkAndGetNextWatermark(T lastElement, long extractedTimestamp)` (for punctuated) is called to determine if a new watermark should be emitted and with which timestamp. -**Note**: If a watermark assigner depends on records read from Kafka to advance its watermarks (which is commonly the case), all topics and partitions need to have a continuous stream of records. Otherwise, the watermarks of the whole application cannot advance and all time-based operations, such as time windows or functions with timers, cannot make progress. A single idle Kafka partition causes this behavior. +**Note**: If a watermark assigner depends on records read from Kafka to advance its watermarks +(which is commonly the case), all topics and partitions need to have a continuous stream of records. +Otherwise, the watermarks of the whole application cannot advance and all time-based operations, +such as time windows or functions with timers, cannot make progress. A single idle Kafka partition causes this behavior. A Flink improvement is planned to prevent this from happening (see [FLINK-5479: Per-partition watermarks in FlinkKafkaConsumer should consider idle partitions]( https://issues.apache.org/jira/browse/FLINK-5479)). @@ -524,7 +547,7 @@ In the meanwhile, a possible workaround is to send *heartbeat messages* to all c ## Kafka Producer -Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). +Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaProducer` for Kafka >= 1.0.0 versions). It allows writing a stream of records to one or more Kafka topics. Example: @@ -637,19 +660,6 @@ we recommend setting the number of retries to a higher value. **Note**: There is currently no transactional producer for Kafka, so Flink can not guarantee exactly-once delivery into a Kafka topic. -
- Attention: Depending on your Kafka configuration, even after Kafka acknowledges - writes you can still experience data loss. In particular keep in mind the following Kafka settings: -
    -
  • acks
    • -
  • log.flush.interval.messages
    • -
  • log.flush.interval.ms
    • -
  • log.flush.*
    • -
- Default values for the above options can easily lead to data loss. Please refer to Kafka documentation - for more explanation. -
- #### Kafka 0.11 and newer With Flink's checkpointing enabled, the `FlinkKafkaProducer011` (`FlinkKafkaProducer` for Kafka >= 1.0.0 versions) can provide @@ -667,21 +677,6 @@ chosen by passing appropriate `semantic` parameter to the `FlinkKafkaProducer011 or `read_uncommitted` - the latter one is the default value) for any application consuming records from Kafka. -
- Attention: Depending on your Kafka configuration, even after Kafka acknowledges - writes you can still experience data losses. In particular keep in mind about following properties - in Kafka config: -
    -
  • acks
    • -
  • log.flush.interval.messages
    • -
  • log.flush.interval.ms
    • -
  • log.flush.*
    • -
- Default values for the above options can easily lead to data loss. Please refer to the Kafka documentation - for more explanation. -
- - ##### Caveats `Semantic.EXACTLY_ONCE` mode relies on the ability to commit transactions @@ -729,7 +724,8 @@ application before first checkpoint completes, by factor larger then `FlinkKafka ## Using Kafka timestamps and Flink event time in Kafka 0.10 -Since Apache Kafka 0.10+, Kafka's messages can carry [timestamps](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message), indicating +Since Apache Kafka 0.10+, Kafka's messages can carry +[timestamps](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message), indicating the time the event has occurred (see ["event time" in Apache Flink](../event_time.html)) or the time when the message has been written to the Kafka broker. @@ -789,19 +785,56 @@ Flink provides first-class support through the Kafka connector to authenticate t configured for Kerberos. Simply configure Flink in `flink-conf.yaml` to enable Kerberos authentication for Kafka like so: 1. Configure Kerberos credentials by setting the following - - - `security.kerberos.login.use-ticket-cache`: By default, this is `true` and Flink will attempt to use Kerberos credentials in ticket caches managed by `kinit`. - Note that when using the Kafka connector in Flink jobs deployed on YARN, Kerberos authorization using ticket caches will not work. This is also the case when deploying using Mesos, as authorization using ticket cache is not supported for Mesos deployments. + - `security.kerberos.login.use-ticket-cache`: By default, this is `true` and Flink will attempt to use Kerberos credentials in ticket caches managed by `kinit`. + Note that when using the Kafka connector in Flink jobs deployed on YARN, Kerberos authorization using ticket caches will not work. + This is also the case when deploying using Mesos, as authorization using ticket cache is not supported for Mesos deployments. - `security.kerberos.login.keytab` and `security.kerberos.login.principal`: To use Kerberos keytabs instead, set values for both of these properties. 2. Append `KafkaClient` to `security.kerberos.login.contexts`: This tells Flink to provide the configured Kerberos credentials to the Kafka login context to be used for Kafka authentication. -Once Kerberos-based Flink security is enabled, you can authenticate to Kafka with either the Flink Kafka Consumer or Producer by simply including the following two settings in the provided properties configuration that is passed to the internal Kafka client: +Once Kerberos-based Flink security is enabled, you can authenticate to Kafka with either the Flink Kafka Consumer or Producer +by simply including the following two settings in the provided properties configuration that is passed to the internal Kafka client: - Set `security.protocol` to `SASL_PLAINTEXT` (default `NONE`): The protocol used to communicate to Kafka brokers. When using standalone Flink deployment, you can also use `SASL_SSL`; please see how to configure the Kafka client for SSL [here](https://kafka.apache.org/documentation/#security_configclients). -- Set `sasl.kerberos.service.name` to `kafka` (default `kafka`): The value for this should match the `sasl.kerberos.service.name` used for Kafka broker configurations. A mismatch in service name between client and server configuration will cause the authentication to fail. +- Set `sasl.kerberos.service.name` to `kafka` (default `kafka`): The value for this should match the `sasl.kerberos.service.name` used for Kafka broker configurations. +A mismatch in service name between client and server configuration will cause the authentication to fail. For more information on Flink configuration for Kerberos security, please see [here]({{ site.baseurl}}/ops/config.html). You can also find [here]({{ site.baseurl}}/ops/security-kerberos.html) further details on how Flink internally setups Kerberos-based security. +## Troubleshooting + +
+If you have a problem with Kafka when using Flink, keep in mind that Flink only wraps +KafkaConsumer or +KafkaProducer +and your problem might be independent of Flink and sometimes can be solved by upgrading Kafka brokers, +reconfiguring Kafka brokers or reconfiguring KafkaConsumer or KafkaProducer in Flink. +Some examples of common problems are listed below. +
+ +### Data loss + +Depending on your Kafka configuration, even after Kafka acknowledges +writes you can still experience data loss. In particular keep in mind about the following properties +in Kafka config: + +- `acks` +- `log.flush.interval.messages` +- `log.flush.interval.ms` +- `log.flush.*` + +Default values for the above options can easily lead to data loss. +Please refer to the Kafka documentation for more explanation. + +### UnknownTopicOrPartitionException + +One possible cause of this error is when a new leader election is taking place, +for example after or during restarting a Kafka broker. +This is a retriable exception, so Flink job should be able to restart and resume normal operation. +It also can be circumvented by changing `retries` property in the producer settings. +However this might cause reordering of messages, +which in turn if undesired can be circumvented by setting `max.in.flight.requests.per.connection` to 1. + {% top %} diff --git a/docs/dev/connectors/streamfile_sink.md b/docs/dev/connectors/streamfile_sink.md index aea66c3cc48da..353a1f84c2aaa 100644 --- a/docs/dev/connectors/streamfile_sink.md +++ b/docs/dev/connectors/streamfile_sink.md @@ -24,16 +24,17 @@ under the License. --> This connector provides a Sink that writes partitioned files to filesystems -supported by the Flink `FileSystem` abstraction. Since in streaming the input -is potentially infinite, the streaming file sink writes data into buckets. The -bucketing behaviour is configurable but a useful default is time-based +supported by the [Flink `FileSystem` abstraction]({{ site.baseurl}}/ops/filesystems.html). + +Since in streaming the input is potentially infinite, the streaming file sink writes data +into buckets. The bucketing behaviour is configurable but a useful default is time-based bucketing where we start writing a new bucket every hour and thus get individual files that each contain a part of the infinite output stream. Within a bucket, we further split the output into smaller part files based on a rolling policy. This is useful to prevent individual bucket files from getting too big. This is also configurable but the default policy rolls files based on -file size and a timeout, i.e if no new data was written to a part file. +file size and a timeout, *i.e* if no new data was written to a part file. The `StreamingFileSink` supports both row-wise encoding formats and bulk-encoding formats, such as [Apache Parquet](http://parquet.apache.org). @@ -51,17 +52,14 @@ Basic usage thus looks like this:
{% highlight java %} -import org.apache.flink.api.common.serialization.Encoder; +import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.core.fs.Path; import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; DataStream input = ...; final StreamingFileSink sink = StreamingFileSink - .forRowFormat(new Path(outputPath), (Encoder) (element, stream) -> { - PrintStream out = new PrintStream(stream); - out.println(element.f1); - }) + .forRowFormat(new Path(outputPath), new SimpleStringEncoder<>("UTF-8")) .build(); input.addSink(sink); @@ -70,19 +68,16 @@ input.addSink(sink);
{% highlight scala %} -import org.apache.flink.api.common.serialization.Encoder +import org.apache.flink.api.common.serialization.SimpleStringEncoder import org.apache.flink.core.fs.Path import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink val input: DataStream[String] = ... -final StreamingFileSink[String] sink = StreamingFileSink - .forRowFormat(new Path(outputPath), (element, stream) => { - val out = new PrintStream(stream) - out.println(element.f1) - }) - .build() - +val sink: StreamingFileSink[String] = StreamingFileSink + .forRowFormat(new Path(outputPath), new SimpleStringEncoder[String]("UTF-8")) + .build() + input.addSink(sink) {% endhighlight %} @@ -120,4 +115,24 @@ has static methods for creating a `BulkWriter.Factory` for various types. every checkpoint.
+#### Important Considerations for S3 + +Important Note 1: For S3, the `StreamingFileSink` +supports only the [Hadoop-based](https://hadoop.apache.org/) FileSystem implementation, not +the implementation based on [Presto](https://prestodb.io/). In case your job uses the +`StreamingFileSink` to write to S3 but you want to use the Presto-based one for checkpointing, +it is advised to use explicitly *"s3a://"* (for Hadoop) as the scheme for the target path of +the sink and *"s3p://"* for checkpointing (for Presto). Using *"s3://"* for both the sink +and checkpointing may lead to unpredictable behavior, as both implementations "listen" to that scheme. + +Important Note 2: To guarantee exactly-once semantics while +being efficient, the `StreamingFileSink` uses the [Multi-part Upload](https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html) +feature of S3 (MPU from now on). This feature allows to upload files in independent chunks (thus the "multi-part") +which can be combined into the original file when all the parts of the MPU are successfully uploaded. +For inactive MPUs, S3 supports a bucket lifecycle rule that the user can use to abort multipart uploads +that don't complete within a specified number of days after being initiated. This implies that if you set this rule +aggressively and take a savepoint with some part-files being not fully uploaded, their associated MPUs may time-out +before the job is restarted. This will result in your job not being able to restore from that savepoint as the +pending part-files are no longer there and Flink will fail with an exception as it tries to fetch them and fails. + {% top %} diff --git a/docs/dev/java_lambdas.md b/docs/dev/java_lambdas.md index 4b306ac455d02..0b4a5cc069d7b 100644 --- a/docs/dev/java_lambdas.md +++ b/docs/dev/java_lambdas.md @@ -107,7 +107,7 @@ env.fromElements(1, 2, 3) .map(new MyTuple2Mapper()) .print(); -public static class MyTuple2Mapper extends MapFunction { +public static class MyTuple2Mapper extends MapFunction> { @Override public Tuple2 map(Integer i) { return Tuple2.of(i, i); diff --git a/docs/dev/libs/ml/quickstart.md b/docs/dev/libs/ml/quickstart.md index e056b28b5057f..2e9a7b9505c9c 100644 --- a/docs/dev/libs/ml/quickstart.md +++ b/docs/dev/libs/ml/quickstart.md @@ -153,6 +153,8 @@ A conversion can be done using a simple normalizer mapping function: {% highlight scala %} +import org.apache.flink.ml.math.Vector + def normalizer : LabeledVector => LabeledVector = { lv => LabeledVector(if (lv.label > 0.0) 1.0 else -1.0, lv.vector) } diff --git a/docs/dev/migration.md b/docs/dev/migration.md index f91a3d8dea990..d472653cbb38d 100644 --- a/docs/dev/migration.md +++ b/docs/dev/migration.md @@ -25,6 +25,17 @@ under the License. * This will be replaced by the TOC {:toc} +## Migrating from Flink 1.3+ to Flink 1.7 + +### API changes for serializer snapshots + +This would be relevant mostly for users implementing custom `TypeSerializer`s for their state. + +The old `TypeSerializerConfigSnapshot` abstraction is now deprecated, and will be fully removed in the future +in favor of the new `TypeSerializerSnapshot`. For details and guides on how to migrate, please see +[Migrating from deprecated serializer snapshot APIs before Flink 1.7] +({{ site.baseurl }}/dev/stream/state/custom_serialization.html#migration-from-deprecated-serializer-snapshot-apis-before-Flink-1.7). + ## Migrating from Flink 1.2 to Flink 1.3 There are a few APIs that have been changed since Flink 1.2. Most of the changes are documented in their diff --git a/docs/dev/stream/operators/index.md b/docs/dev/stream/operators/index.md index 422dbbf913634..b89fa898fe794 100644 --- a/docs/dev/stream/operators/index.md +++ b/docs/dev/stream/operators/index.md @@ -434,14 +434,14 @@ IterativeStream iteration = initialStream.iterate(); DataStream iterationBody = iteration.map (/*do something*/); DataStream feedback = iterationBody.filter(new FilterFunction(){ @Override - public boolean filter(Integer value) throws Exception { + public boolean filter(Long value) throws Exception { return value > 0; } }); iteration.closeWith(feedback); DataStream output = iterationBody.filter(new FilterFunction(){ @Override - public boolean filter(Integer value) throws Exception { + public boolean filter(Long value) throws Exception { return value <= 0; } }); diff --git a/docs/dev/stream/operators/joining.md b/docs/dev/stream/operators/joining.md index b95aaddbbaafc..c415f63f7815c 100644 --- a/docs/dev/stream/operators/joining.md +++ b/docs/dev/stream/operators/joining.md @@ -70,7 +70,7 @@ DataStream greenStream = ... orangeStream.join(greenStream) .where() .equalTo() - .window(TumblingEventTimeWindows.of(Time.seconds(2))) + .window(TumblingEventTimeWindows.of(Time.milliseconds(2))) .apply (new JoinFunction (){ @Override public String join(Integer first, Integer second) { @@ -101,7 +101,7 @@ orangeStream.join(greenStream)
## Sliding Window Join -When performing a sliding window join, all elements with a common key and common sliding window are joined are pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Elements of one stream that do not have elements from the other stream in the current sliding window are not emitted! Note that some elements might be joined in one sliding window but not in another! +When performing a sliding window join, all elements with a common key and common sliding window are joined as pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Elements of one stream that do not have elements from the other stream in the current sliding window are not emitted! Note that some elements might be joined in one sliding window but not in another! diff --git a/docs/dev/stream/state/custom_serialization.md b/docs/dev/stream/state/custom_serialization.md index f3941bc0c982a..cc727bef6c2d1 100644 --- a/docs/dev/stream/state/custom_serialization.md +++ b/docs/dev/stream/state/custom_serialization.md @@ -1,8 +1,8 @@ --- title: "Custom Serialization for Managed State" -nav-title: "Custom Serialization" +nav-title: "Custom State Serialization" nav-parent_id: streaming_state -nav-pos: 6 +nav-pos: 7 --- -If your application uses Flink's managed state, it might be necessary to implement custom serialization logic for special use cases. +* ToC +{:toc} -This page is targeted as a guideline for users who require the use of custom serialization for their state, covering how -to provide a custom serializer and how to handle upgrades to the serializer for compatibility. If you're simply using -Flink's own serializers, this page is irrelevant and can be skipped. +This page is targeted as a guideline for users who require the use of custom serialization for their state, covering +how to provide a custom state serializer as well as guidelines and best practices for implementing serializers that allow +state schema evolution. -### Using custom serializers +If you're simply using Flink's own serializers, this page is irrelevant and can be ignored. -As demonstrated in the above examples, when registering a managed operator or keyed state, a `StateDescriptor` is required +## Using custom state serializers + +When registering a managed operator or keyed state, a `StateDescriptor` is required to specify the state's name, as well as information about the type of the state. The type information is used by Flink's [type serialization framework](../../types_serialization.html) to create appropriate serializers for the state. @@ -66,125 +69,204 @@ checkpointedState = getRuntimeContext.getListState(descriptor) -Note that Flink writes state serializers along with the state as metadata. In certain cases on restore (see following -subsections), the written serializer needs to be deserialized and used. Therefore, it is recommended to avoid using -anonymous classes as your state serializers. Anonymous classes do not have a guarantee on the generated classname, -which varies across compilers and depends on the order that they are instantiated within the enclosing class, which can -easily cause the previously written serializer to be unreadable (since the original class can no longer be found in the -classpath). - -### Handling serializer upgrades and compatibility - -Flink allows changing the serializers used to read and write managed state, so that users are not locked in to any -specific serialization. When state is restored, the new serializer registered for the state (i.e., the serializer -that comes with the `StateDescriptor` used to access the state in the restored job) will be checked for compatibility, -and is replaced as the new serializer for the state. - -A compatible serializer would mean that the serializer is capable of reading previous serialized bytes of the state, -and the written binary format of the state also remains identical. The means to check the new serializer's compatibility -is provided through the following two methods of the `TypeSerializer` interface: - -{% highlight java %} -public abstract TypeSerializerConfigSnapshot snapshotConfiguration(); -public abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); -{% endhighlight %} - -Briefly speaking, every time a checkpoint is performed, the `snapshotConfiguration` method is called to create a -point-in-time view of the state serializer's configuration. The returned configuration snapshot is stored along with the -checkpoint as the state's metadata. When the checkpoint is used to restore a job, that serializer configuration snapshot -will be provided to the _new_ serializer of the same state via the counterpart method, `ensureCompatibility`, to verify -compatibility of the new serializer. This method serves as a check for whether or not the new serializer is compatible, -as well as a hook to possibly reconfigure the new serializer in the case that it is incompatible. +## State serializers and schema evolution -Note that Flink's own serializers are implemented such that they are at least compatible with themselves, i.e. when the -same serializer is used for the state in the restored job, the serializer's will reconfigure themselves to be compatible -with their previous configuration. +This section explains the user-facing abstractions related to state serialization and schema evolution, and necessary +internal details about how Flink interacts with these abstractions. -The following subsections illustrate guidelines to implement these two methods when using custom serializers. +When restoring from savepoints, Flink allows changing the serializers used to read and write previously registered state, +so that users are not locked in to any specific serialization schema. When state is restored, a new serializer will be +registered for the state (i.e., the serializer that comes with the `StateDescriptor` used to access the state in the +restored job). This new serializer may have a different schema than that of the previous serializer. Therefore, when +implementing state serializers, besides the basic logic of reading / writing data, another important thing to keep in +mind is how the serialization schema can be changed in the future. -#### Implementing the `snapshotConfiguration` method +When speaking of *schema*, in this context the term is interchangeable between referring to the *data model* of a state +type and the *serialized binary format* of a state type. The schema, generally speaking, can change for a few cases: -The serializer's configuration snapshot should capture enough information such that on restore, the information -carried over to the new serializer for the state is sufficient for it to determine whether or not it is compatible. -This could typically contain information about the serializer's parameters or binary format of the serialized data; -generally, anything that allows the new serializer to decide whether or not it can be used to read previous serialized -bytes, and that it writes in the same binary format. + 1. Data schema of the state type has evolved, i.e. adding or removing a field from a POJO that is used as state. + 2. Generally speaking, after a change to the data schema, the serialization format of the serializer will need to be upgraded. + 3. Configuration of the serializer has changed. + +In order for the new execution to have information about the *written schema* of state and detect whether or not the +schema has changed, upon taking a savepoint of an operator's state, a *snapshot* of the state serializer needs to be +written along with the state bytes. This is abstracted a `TypeSerializerSnapshot`, explained in the next subsection. -How the serializer's configuration snapshot is written to and read from checkpoints is fully customizable. The below -is the base class for all serializer configuration snapshot implementations, the `TypeSerializerConfigSnapshot`. +### The `TypeSerializerSnapshot` abstraction +
{% highlight java %} -public abstract TypeSerializerConfigSnapshot extends VersionedIOReadableWritable { - public abstract int getVersion(); - public void read(DataInputView in) {...} - public void write(DataOutputView out) {...} +public interface TypeSerializerSnapshot { + int getCurrentVersion(); + void writeSnapshot(DataOuputView out) throws IOException; + void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException; + TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer); + TypeSerializer restoreSerializer(); } {% endhighlight %} +
-The `read` and `write` methods define how the configuration is read from and written to the checkpoint. The base -implementations contain logic to read and write the version of the configuration snapshot, so it should be extended and -not completely overridden. - -The version of the configuration snapshot is determined through the `getVersion` method. Versioning for the serializer -configuration snapshot is the means to maintain compatible configurations, as information included in the configuration -may change over time. By default, configuration snapshots are only compatible with the current version (as returned by -`getVersion`). To indicate that the configuration is compatible with other versions, override the `getCompatibleVersions` -method to return more version values. When reading from the checkpoint, you can use the `getReadVersion` method to -determine the version of the written configuration and adapt the read logic to the specific version. - -Attention The version of the serializer's configuration snapshot is **not** -related to upgrading the serializer. The exact same serializer can have different implementations of its -configuration snapshot, for example when more information is added to the configuration to allow more comprehensive -compatibility checks in the future. - -One limitation of implementing a `TypeSerializerConfigSnapshot` is that an empty constructor must be present. The empty -constructor is required when reading the configuration snapshot from checkpoints. - -#### Implementing the `ensureCompatibility` method - -The `ensureCompatibility` method should contain logic that performs checks against the information about the previous -serializer carried over via the provided `TypeSerializerConfigSnapshot`, basically doing one of the following: - - * Check whether the serializer is compatible, while possibly reconfiguring itself (if required) so that it may be - compatible. Afterwards, acknowledge with Flink that the serializer is compatible. - - * Acknowledge that the serializer is incompatible and that state migration is required before Flink can proceed with - using the new serializer. - -The above cases can be translated to code by returning one of the following from the `ensureCompatibility` method: - - * **`CompatibilityResult.compatible()`**: This acknowledges that the new serializer is compatible, or has been reconfigured to - be compatible, and Flink can proceed with the job with the serializer as is. - - * **`CompatibilityResult.requiresMigration()`**: This acknowledges that the serializer is incompatible, or cannot be - reconfigured to be compatible, and requires a state migration before the new serializer can be used. State migration - is performed by using the previous serializer to read the restored state bytes to objects, and then serialized again - using the new serializer. - - * **`CompatibilityResult.requiresMigration(TypeDeserializer deserializer)`**: This acknowledgement has equivalent semantics - to `CompatibilityResult.requiresMigration()`, but in the case that the previous serializer cannot be found or loaded - to read the restored state bytes for the migration, a provided `TypeDeserializer` can be used as a fallback resort. - -Attention Currently, as of Flink 1.3, if the result of the compatibility check -acknowledges that state migration needs to be performed, the job simply fails to restore from the checkpoint as state -migration is currently not available. The ability to migrate state will be introduced in future releases. - -### Managing `TypeSerializer` and `TypeSerializerConfigSnapshot` classes in user code - -Since `TypeSerializer`s and `TypeSerializerConfigSnapshot`s are written as part of checkpoints along with the state -values, the availability of the classes within the classpath may affect restore behaviour. - -`TypeSerializer`s are directly written into checkpoints using Java Object Serialization. In the case that the new -serializer acknowledges that it is incompatible and requires state migration, it will be required to be present to be -able to read the restored state bytes. Therefore, if the original serializer class no longer exists or has been modified -(resulting in a different `serialVersionUID`) as a result of a serializer upgrade for the state, the restore would -not be able to proceed. The alternative to this requirement is to provide a fallback `TypeDeserializer` when -acknowledging that state migration is required, using `CompatibilityResult.requiresMigration(TypeDeserializer deserializer)`. +
+{% highlight java %} +public abstract class TypeSerializer { + + // ... + + public abstract TypeSerializerSnapshot snapshotConfiguration(); +} +{% endhighlight %} +
-The class of `TypeSerializerConfigSnapshot`s in the restored checkpoint must exist in the classpath, as they are -fundamental components to compatibility checks on upgraded serializers and would not be able to be restored if the class -is not present. Since configuration snapshots are written to checkpoints using custom serialization, the implementation -of the class is free to be changed, as long as compatibility of the configuration change is handled using the versioning -mechanisms in `TypeSerializerConfigSnapshot`. +A serializer's `TypeSerializerSnapshot` is a point-in-time information that serves as the single source of truth about +the state serializer's write schema, as well as any additional information mandatory to restore a serializer that +would be identical to the given point-in-time. The logic about what should be written and read at restore time +as the serializer snapshot is defined in the `writeSnapshot` and `readSnapshot` methods. + +Note that the snapshot's own write schema may also need to change over time (e.g. when you wish to add more information +about the serializer to the snapshot). To facilitate this, snapshots are versioned, with the current version +number defined in the `getCurrentVersion` method. On restore, when the serializer snapshot is read from savepoints, +the version of the schema in which the snapshot was written in will be provided to the `readSnapshot` method so that +the read implementation can handle different versions. + +At restore time, the logic that detects whether or not the new serializer's schema has changed should be implemented in +the `resolveSchemaCompatibility` method. When previous registered state is registered again with new serializers in the +restored execution of an operator, the new serializer is provided to the previous serializer's snapshot via this method. +This method returns a `TypeSerializerSchemaCompatibility` representing the result of the compatibility resolution, +which can be one of the following: + + 1. **`TypeSerializerSchemaCompatibility.compatibleAsIs()`**: this result signals that the new serializer is compatible, + meaning that the new serializer has identical schema with the previous serializer. It is possible that the new + serializer has been reconfigured in the `resolveSchemaCompatibility` method so that it is compatible. + 2. **`TypeSerializerSchemaCompatibility.compatibleAfterMigration()`**: this result signals that the new serializer has a + different serialization schema, and it is possible to migrate from the old schema by using the previous serializer + (which recognizes the old schema) to read bytes into state objects, and then rewriting the object back to bytes with + the new serializer (which recognizes the new schema). + 3. **`TypeSerializerSchemaCompatibility.incompatible()`**: this result signals that the new serializer has a + different serialization schema, but it is not possible to migrate from the old schema. + +The last bit of detail is how the previous serializer is obtained in the case that migration is required. +Another important role of a serializer's `TypeSerializerSnapshot` is that it serves as a factory to restore +the previous serializer. More specifically, the `TypeSerializerSnapshot` should implement the `restoreSerializer` method +to instantiate a serializer instance that recognizes the previous serializer's schema and configuration, and can therefore +safely read data written by the previous serializer. + +### How Flink interacts with the `TypeSerializer` and `TypeSerializerSnapshot` abstractions + +To wrap up, this section concludes how Flink, or more specifically the state backends, interact with the +abstractions. The interaction is slightly different depending on the state backend, but this is orthogonal +to the implementation of state serializers and their serializer snapshots. + +#### Off-heap state backends (e.g. `RocksDBStateBackend`) + + 1. **Register new state with a state serializer that has schema _A_** + - the registered `TypeSerializer` for the state is used to read / write state on every state access. + - State is written in schema *A*. + 2. **Take a savepoint** + - The serializer snapshot is extracted via the `TypeSerializer#snapshotConfiguration` method. + - The serializer snapshot is written to the savepoint, as well as the already-serialized state bytes (with schema *A*). + 3. **Restored execution re-accesses restored state bytes with new state serializer that has schema _B_** + - The previous state serializer's snapshot is restored. + - State bytes are not deserialized on restore, only loaded back to the state backends (therefore, still in schema *A*). + - Upon receiving the new serializer, it is provided to the restored previous serializer's snapshot via the + `TypeSerializer#resolveSchemaCompatibility` to check for schema compatibility. + 4. **Migrate state bytes in backend from schema _A_ to schema _B_** + - If the compatibility resolution reflects that the schema has changed and migration is possible, schema migration is + performed. The previous state serializer which recognizes schema _A_ will be obtained from the serializer snapshot, via + `TypeSerializerSnapshot#restoreSerializer()`, and is used to deserialize state bytes to objects, which in turn + are re-written again with the new serializer, which recognizes schema _B_ to complete the migration. All entries + of the accessed state is migrated all-together before processing continues. + - If the resolution signals incompatibility, then the state access fails with an exception. + +#### Heap state backends (e.g. `MemoryStateBackend`, `FsStateBackend`) + + 1. **Register new state with a state serializer that has schema _A_** + - the registered `TypeSerializer` is maintained by the state backend. + 2. **Take a savepoint, serializing all state with schema _A_** + - The serializer snapshot is extracted via the `TypeSerializer#snapshotConfiguration` method. + - The serializer snapshot is written to the savepoint. + - State objects are now serialized to the savepoint, written in schema _A_. + 3. **On restore, deserialize state into objects in heap** + - The previous state serializer's snapshot is restored. + - The previous serializer, which recognizes schema _A_, is obtained from the serializer snapshot, via + `TypeSerializerSnapshot#restoreSerializer()`, and is used to deserialize state bytes to objects. + - From now on, all of the state is already deserialized. + 4. **Restored execution re-accesses previous state with new state serializer that has schema _B_** + - Upon receiving the new serializer, it is provided to the restored previous serializer's snapshot via the + `TypeSerializer#resolveSchemaCompatibility` to check for schema compatibility. + - If the compatibility check signals that migration is required, nothing happens in this case since for + heap backends, all state is already deserialized into objects. + - If the resolution signals incompatibility, then the state access fails with an exception. + 5. **Take another savepoint, serializing all state with schema _B_** + - Same as step 2., but now state bytes are all in schema _B_. + +## Implementation notes and best practices + +#### 1. Flink restores serializer snapshots by instantiating them with their classname + +A serializer's snapshot, being the single source of truth for how a registered state was serialized, serves as an +entry point to reading state in savepoints. In order to be able to restore and access previous state, the previous state +serializer's snapshot must be able to be restored. + +Flink restores serializer snapshots by first instantiating the `TypeSerializerSnapshot` with its classname (written +along with the snapshot bytes). Therefore, to avoid being subject to unintended classname changes or instantiation +failures, `TypeSerializerSnapshot` classes should: + + - avoid being implemented as anonymous classes or nested classes, + - have a public, nullary constructor for instantiation + +#### 2. Avoid sharing the same `TypeSerializerSnapshot` class across different serializers + +Since schema compatibility checks goes through the serializer snapshots, having multiple serializers returning +the same `TypeSerializerSnapshot` class as their snapshot would complicate the implementation for the +`TypeSerializerSnapshot#resolveSchemaCompatibility` and `TypeSerializerSnapshot#restoreSerializer()` method. + +This would also be a bad separation of concerns; a single serializer's serialization schema, +configuration, as well as how to restore it, should be consolidated in its own dedicated `TypeSerializerSnapshot` class. + +#### 3. Use the `CompositeSerializerSnapshot` utility for serializers that contain nested serializers + +There may be cases where a `TypeSerializer` relies on other nested `TypeSerializer`s; take for example Flink's +`TupleSerializer`, where it is configured with nested `TypeSerializer`s for the tuple fields. In this case, +the snapshot of the most outer serializer should also contain snapshots of the nested serializers. + +The `CompositeSerializerSnapshot` can be used specifically for this scenario. It wraps the logic of resolving +the overall schema compatibility check result for the composite serializer. +For an example of how it should be used, one can refer to Flink's +[ListSerializerSnapshot](https://github.com/apache/flink/blob/master/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java) implementation. + +## Migrating from deprecated serializer snapshot APIs before Flink 1.7 + +This section is a guide for API migration from serializers and serializer snapshots that existed before Flink 1.7. + +Before Flink 1.7, serializer snapshots were implemented as a `TypeSerializerConfigSnapshot` (which is now deprecated, +and will eventually be removed in the future to be fully replaced by the new `TypeSerializerSnapshot` interface). +Moreover, the responsibility of serializer schema compatibility checks lived within the `TypeSerializer`, +implemented in the `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` method. + +Another major difference between the new and old abstractions is that the deprecated `TypeSerializerConfigSnapshot` +did not have the capability of instantiating the previous serializer. Therefore, in the case where your serializer +still returns a subclass of `TypeSerializerConfigSnapshot` as its snapshot, the serializer instance itself will always +be written to savepoints using Java serialization so that the previous serializer may be available at restore time. +This is very undesirable, since whether or not restoring the job will be successful is susceptible to availability +of the previous serializer's class, or in general, whether or not the serializer instance can be read back at restore +time using Java serialization. This means that you be limited to the same serializer for your state, +and could be problematic once you want to upgrade serializer classes or perform schema migration. + +To be future-proof and have flexibility to migrate your state serializers and schema, it is highly recommended to +migrate from the old abstractions. The steps to do this is as follows: + + 1. Implement a new subclass of `TypeSerializerSnapshot`. This will be the new snapshot for your serializer. + 2. Return the new `TypeSerializerSnapshot` as the serializer snapshot for your serializer in the + `TypeSerializer#snapshotConfiguration()` method. + 3. Restore the job from the savepoint that existed before Flink 1.7, and then take a savepoint again. + Note that at this step, the old `TypeSerializerConfigSnapshot` of the serializer must still exist in the classpath, + and the implementation for the `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` method must not be + removed. The purpose of this process is to replace the `TypeSerializerConfigSnapshot` written in old savepoints + with the newly implemented `TypeSerializerSnapshot` for the serializer. + 4. Once you have a savepoint taken with Flink 1.7, the savepoint will contain `TypeSerializerSnapshot` as the + state serializer snapshot, and the serializer instance will no longer be written in the savepoint. + At this point, it is now safe to remove all implementations of the old abstraction (remove the old + `TypeSerializerConfigSnapshot` implementation as will as the + `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` from the serializer). {% top %} diff --git a/docs/dev/stream/state/index.md b/docs/dev/stream/state/index.md index f9a24c3d0399d..10de348a76d25 100644 --- a/docs/dev/stream/state/index.md +++ b/docs/dev/stream/state/index.md @@ -52,6 +52,7 @@ Where to go next? * [The Broadcast State Pattern](broadcast_state.html): Explains how to connect a broadcast stream with a non-broadcast stream and use state to exchange information between them. * [Checkpointing](checkpointing.html): Describes how to enable and configure checkpointing for fault tolerance. * [Queryable State](queryable_state.html): Explains how to access state from outside of Flink during runtime. -* [Custom Serialization for Managed State](custom_serialization.html): Discusses custom serialization logic for state and its upgrades. +* [State Schema Evolution](schema_evolution.html): Shows how schema of state types can be evolved. +* [Custom Serialization for Managed State](custom_serialization.html): Discusses how to implement custom serializers, especially for schema evolution. {% top %} diff --git a/docs/dev/stream/state/schema_evolution.md b/docs/dev/stream/state/schema_evolution.md new file mode 100644 index 0000000000000..f2f52b22ad74f --- /dev/null +++ b/docs/dev/stream/state/schema_evolution.md @@ -0,0 +1,93 @@ +--- +title: "State Schema Evolution" +nav-parent_id: streaming_state +nav-pos: 6 +--- + + +* ToC +{:toc} + +Apache Flink streaming applications are typically designed to run indefinitely or for long periods of time. +As with all long-running services, the applications need to be updated to adapt to changing requirements. +This goes the same for data schemas that the applications work against; they evolve along with the application. + +This page provides an overview of how you can evolve your state type's data schema. +The current restrictions varies across different types and state structures (`ValueState`, `ListState`, etc.). + +Note that the information on this page is relevant only if you are using state serializers that are +generated by Flink's own [type serialization framework]({{ site.baseurl }}/dev/types_serialization.html). +That is, when declaring your state, the provided state descriptor is not configured to use a specific `TypeSerializer` +or `TypeInformation`, in which case Flink infers information about the state type: + +
+{% highlight java %} +ListStateDescriptor descriptor = + new ListStateDescriptor<>( + "state-name", + MyPojoType.class); + +checkpointedState = getRuntimeContext().getListState(descriptor); +{% endhighlight %} +
+ +Under the hood, whether or not the schema of state can be evolved depends on the serializer used to read / write +persisted state bytes. Simply put, a registered state's schema can only be evolved if its serializer properly +supports it. This is handled transparently by serializers generated by Flink's type serialization framework +(current scope of support is listed [below]({{ site.baseurl }}/dev/stream/state/schema_evolution.html#supported-data-types-for-schema-evolution)). + +If you intend to implement a custom `TypeSerializer` for your state type and would like to learn how to implement +the serializer to support state schema evolution, please refer to +[Custom State Serialization]({{ site.baseurl }}/dev/stream/state/custom_serialization.html). +The documentation there also covers necessary internal details about the interplay between state serializers and Flink's +state backends to support state schema evolution. + +## Evolving state schema + +To evolve the schema of a given state type, you would take the following steps: + + 1. Take a savepoint of your Flink streaming job. + 2. Update state types in your application (e.g., modifying your Avro type schema). + 3. Restore the job from the savepoint. When accessing state for the first time, Flink will assess whether or not + the schema had been changed for the state, and migrate state schema if necessary. + +The process of migrating state to adapt to changed schemas happens automatically, and independently for each state. +This process is performed internally by Flink by first checking if the new serializer for the state has different +serialization schema than the previous serializer; if so, the previous serializer is used to read the state to objects, +and written back to bytes again with the new serializer. + +Further details about the migration process is out of the scope of this documentation; please refer to +[here]({{ site.baseurl }}/dev/stream/state/custom_serialization.html). + +## Supported data types for schema evolution + +Currently, schema evolution is supported only for Avro. Therefore, if you care about schema evolution for +state, it is currently recommended to always use Avro for state data types. + +There are plans to extend the support for more composite types, such as POJOs; for more details, +please refer to [FLINK-10897](https://issues.apache.org/jira/browse/FLINK-10897). + +### Avro types + +Flink fully supports evolving schema of Avro type state, as long as the schema change is considered compatible by +[Avro's rules for schema resolution](http://avro.apache.org/docs/current/spec.html#Schema+Resolution). + +One limitation is that Avro generated classes used as the state type cannot be relocated or have different +namespaces when the job is restored. diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md index decf1dbe9de25..a3109f2c2fa4b 100644 --- a/docs/dev/stream/state/state.md +++ b/docs/dev/stream/state/state.md @@ -142,7 +142,7 @@ is available in a `RichFunction` has these methods for accessing state: * `ValueState getState(ValueStateDescriptor)` * `ReducingState getReducingState(ReducingStateDescriptor)` * `ListState getListState(ListStateDescriptor)` -* `AggregatingState getAggregatingState(AggregatingState)` +* `AggregatingState getAggregatingState(AggregatingStateDescriptor)` * `FoldingState getFoldingState(FoldingStateDescriptor)` * `MapState getMapState(MapStateDescriptor)` diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md index 5f1112706cee7..e7f9263c29d60 100644 --- a/docs/dev/table/connect.md +++ b/docs/dev/table/connect.md @@ -40,14 +40,15 @@ The following table list all available connectors and formats. Their mutual comp ### Connectors -| Name | Version | Maven dependency | SQL Client JAR | -| :---------------- | :------------ | :--------------------------- | :----------------------| -| Filesystem | | Built-in | Built-in | -| Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.8 | `flink-connector-kafka-0.8` | Not available | -| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.11 | `flink-connector-kafka-0.11` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.11{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.11{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Name | Version | Maven dependency | SQL Client JAR | +| :---------------- | :------------------ | :--------------------------- | :----------------------| +| Filesystem | | Built-in | Built-in | +| Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Apache Kafka | 0.8 | `flink-connector-kafka-0.8` | Not available | +| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Apache Kafka | 0.11 | `flink-connector-kafka-0.11` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.11{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.11{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Apache Kafka | 0.11+ (`universal`) | `flink-connector-kafka` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | ### Formats @@ -153,7 +154,7 @@ tableEnvironment new Schema() .field("rowtime", Types.SQL_TIMESTAMP) .rowtime(new Rowtime() - .timestampsFromField("ts") + .timestampsFromField("timestamp") .watermarksPeriodicBounded(60000) ) .field("user", Types.LONG) @@ -311,7 +312,7 @@ The following timestamp extractors are supported: .timestampsFromField("ts_field") // required: original field name in the input ) -// Converts the assigned timestamps from a DataStream API record into the rowtime attribute +// Converts the assigned timestamps from a DataStream API record into the rowtime attribute // and thus preserves the assigned timestamps from the source. // This requires a source that assigns timestamps (e.g., Kafka 0.10+). .rowtime( @@ -336,7 +337,7 @@ rowtime: type: from-field from: "ts_field" # required: original field name in the input -# Converts the assigned timestamps from a DataStream API record into the rowtime attribute +# Converts the assigned timestamps from a DataStream API record into the rowtime attribute # and thus preserves the assigned timestamps from the source. rowtime: timestamps: @@ -350,7 +351,7 @@ The following watermark strategies are supported:
{% highlight java %} -// Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum +// Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum // observed timestamp so far minus 1. Rows that have a timestamp equal to the max timestamp // are not late. .rowtime( @@ -376,7 +377,7 @@ The following watermark strategies are supported:
{% highlight yaml %} -# Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum +# Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum # observed timestamp so far minus 1. Rows that have a timestamp equal to the max timestamp # are not late. rowtime: @@ -524,7 +525,8 @@ The Kafka connector allows for reading and writing from and to an Apache Kafka t {% highlight java %} .connect( new Kafka() - .version("0.11") // required: valid connector versions are "0.8", "0.9", "0.10", and "0.11" + .version("0.11") // required: valid connector versions are + // "0.8", "0.9", "0.10", "0.11", and "universal" .topic("...") // required: topic name from which the table is read // optional: connector specific properties @@ -549,7 +551,8 @@ The Kafka connector allows for reading and writing from and to an Apache Kafka t {% highlight yaml %} connector: type: kafka - version: "0.11" # required: valid connector versions are "0.8", "0.9", "0.10", and "0.11" + version: "0.11" # required: valid connector versions are + # "0.8", "0.9", "0.10", "0.11", and "universal" topic: ... # required: topic name from which the table is read properties: # optional: connector specific properties @@ -583,7 +586,9 @@ connector: **Consistency guarantees:** By default, a Kafka sink ingests data with at-least-once guarantees into a Kafka topic if the query is executed with [checkpointing enabled]({{ site.baseurl }}/dev/stream/state/checkpointing.html#enabling-and-configuring-checkpointing). -**Kafka 0.10+ Timestamps:** Since Kafka 0.10, Kafka messages have a timestamp as metadata that specifies when the record was written into the Kafka topic. These timestamps can be used for a [rowtime attribute](connect.html#defining-the-schema) by selecting `timestamps: from-source` in YAML and `timestampsFromSource()` in Java/Scala respectively. +**Kafka 0.10+ Timestamps:** Since Kafka 0.10, Kafka messages have a timestamp as metadata that specifies when the record was written into the Kafka topic. These timestamps can be used for a [rowtime attribute](connect.html#defining-the-schema) by selecting `timestamps: from-source` in YAML and `timestampsFromSource()` in Java/Scala respectively. + +**Kafka 0.11+ Versioning:** Since Flink 1.7, the Kafka connector definition should be independent of a hard-coded Kafka version. Use the connector version `universal` as a wildcard for Flink's Kafka connector that is compatible with all Kafka versions starting from 0.11. Make sure to add the version-specific Kafka dependency. In addition, a corresponding format needs to be specified for reading and writing rows from and to Kafka. @@ -690,7 +695,7 @@ connector: **Key extraction:** Flink automatically extracts valid keys from a query. For example, a query `SELECT a, b, c FROM t GROUP BY a, b` defines a composite key of the fields `a` and `b`. The Elasticsearch connector generates a document ID string for every row by concatenating all key fields in the order defined in the query using a key delimiter. A custom representation of null literals for key fields can be defined. -Attention A JSON format defines how to encode documents for the external system, therefore, it must be added as a [dependency](connect.html#formats). +Attention A JSON format defines how to encode documents for the external system, therefore, it must be added as a [dependency](connect.html#formats). {% top %} @@ -712,8 +717,8 @@ The CSV format allows to read and write comma-separated rows. new Csv() .field("field1", Types.STRING) // required: ordered format fields .field("field2", Types.TIMESTAMP) - .fieldDelimiter(",") // optional: string delimiter "," by default - .lineDelimiter("\n") // optional: string delimiter "\n" by default + .fieldDelimiter(",") // optional: string delimiter "," by default + .lineDelimiter("\n") // optional: string delimiter "\n" by default .quoteCharacter('"') // optional: single character for string values, empty by default .commentPrefix('#') // optional: string to indicate comments, empty by default .ignoreFirstLine() // optional: ignore the first line, by default it is not skipped @@ -731,8 +736,8 @@ format: type: VARCHAR - name: field2 type: TIMESTAMP - field-delimiter: "," # optional: string delimiter "," by default - line-delimiter: "\n" # optional: string delimiter "\n" by default + field-delimiter: "," # optional: string delimiter "," by default + line-delimiter: "\n" # optional: string delimiter "\n" by default quote-character: '"' # optional: single character for string values, empty by default comment-prefix: '#' # optional: string to indicate comments, empty by default ignore-first-line: false # optional: boolean flag to ignore the first line, by default it is not skipped @@ -987,7 +992,7 @@ These are the additional `TableSink`s which are provided with Flink: | **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description** | `CsvTableSink` | `flink-table` | Y | Append | A simple sink for CSV files. | `JDBCAppendTableSink` | `flink-jdbc` | Y | Append | Writes a Table to a JDBC table. -| `CassandraAppendTableSink` | `flink-connector-cassandra` | N | Append | Writes a Table to a Cassandra table. +| `CassandraAppendTableSink` | `flink-connector-cassandra` | N | Append | Writes a Table to a Cassandra table. ### OrcTableSource @@ -1039,7 +1044,7 @@ val orcTableSource = OrcTableSource.builder() ### CsvTableSink -The `CsvTableSink` emits a `Table` to one or more CSV files. +The `CsvTableSink` emits a `Table` to one or more CSV files. The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming/dynamic_tables.html#table-to-stream-conversion) for details. When emitting a streaming table, rows are written at least once (if checkpointing is enabled) and the `CsvTableSink` does not split output files into bucket files but continuously writes to the same files. @@ -1048,17 +1053,17 @@ The sink only supports append-only streaming tables. It cannot be used to emit a {% highlight java %} CsvTableSink sink = new CsvTableSink( - path, // output path + path, // output path "|", // optional: delimit files by '|' 1, // optional: write to a single file WriteMode.OVERWRITE); // optional: override existing files tableEnv.registerTableSink( "csvOutputTable", - sink, // specify table schema new String[]{"f0", "f1"}, - new TypeInformation[]{Types.STRING, Types.INT}); + new TypeInformation[]{Types.STRING, Types.INT}, + sink); Table table = ... table.insertInto("csvOutputTable"); @@ -1069,17 +1074,17 @@ table.insertInto("csvOutputTable"); {% highlight scala %} val sink: CsvTableSink = new CsvTableSink( - path, // output path + path, // output path fieldDelim = "|", // optional: delimit files by '|' numFiles = 1, // optional: write to a single file writeMode = WriteMode.OVERWRITE) // optional: override existing files tableEnv.registerTableSink( "csvOutputTable", - sink, // specify table schema Array[String]("f0", "f1"), - Array[TypeInformation[_]](Types.STRING, Types.INT)) + Array[TypeInformation[_]](Types.STRING, Types.INT), + sink) val table: Table = ??? table.insertInto("csvOutputTable") @@ -1108,10 +1113,10 @@ JDBCAppendTableSink sink = JDBCAppendTableSink.builder() tableEnv.registerTableSink( "jdbcOutputTable", - sink, // specify table schema new String[]{"id"}, - new TypeInformation[]{Types.INT}); + new TypeInformation[]{Types.INT}, + sink); Table table = ... table.insertInto("jdbcOutputTable"); @@ -1129,10 +1134,10 @@ val sink: JDBCAppendTableSink = JDBCAppendTableSink.builder() tableEnv.registerTableSink( "jdbcOutputTable", - sink, // specify table schema Array[String]("id"), - Array[TypeInformation[_]](Types.INT)) + Array[TypeInformation[_]](Types.INT), + sink) val table: Table = ??? table.insertInto("jdbcOutputTable") @@ -1140,7 +1145,7 @@ table.insertInto("jdbcOutputTable")
-Similar to using JDBCOutputFormat, you have to explicitly specify the name of the JDBC driver, the JDBC URL, the query to be executed, and the field types of the JDBC table. +Similar to using JDBCOutputFormat, you have to explicitly specify the name of the JDBC driver, the JDBC URL, the query to be executed, and the field types of the JDBC table. {% top %} @@ -1159,16 +1164,16 @@ To use the `CassandraAppendTableSink`, you have to add the Cassandra connector d ClusterBuilder builder = ... // configure Cassandra cluster connection CassandraAppendTableSink sink = new CassandraAppendTableSink( - builder, + builder, // the query must match the schema of the table - INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)); + "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)"); tableEnv.registerTableSink( "cassandraOutputTable", - sink, // specify table schema new String[]{"id", "name", "value"}, - new TypeInformation[]{Types.INT, Types.STRING, Types.DOUBLE}); + new TypeInformation[]{Types.INT, Types.STRING, Types.DOUBLE}, + sink); Table table = ... table.insertInto(cassandraOutputTable); @@ -1180,16 +1185,16 @@ table.insertInto(cassandraOutputTable); val builder: ClusterBuilder = ... // configure Cassandra cluster connection val sink: CassandraAppendTableSink = new CassandraAppendTableSink( - builder, + builder, // the query must match the schema of the table - INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)) + "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)") tableEnv.registerTableSink( "cassandraOutputTable", - sink, // specify table schema Array[String]("id", "name", "value"), - Array[TypeInformation[_]](Types.INT, Types.STRING, Types.DOUBLE)) + Array[TypeInformation[_]](Types.INT, Types.STRING, Types.DOUBLE), + sink) val table: Table = ??? table.insertInto(cassandraOutputTable) diff --git a/docs/dev/table/functions.md b/docs/dev/table/functions.md index e47b31772b38d..f41139d14957c 100644 --- a/docs/dev/table/functions.md +++ b/docs/dev/table/functions.md @@ -3487,7 +3487,7 @@ DATE_FORMAT(timestamp, string) {% endhighlight %} -

Returns a string that formats timestamp with a specified format string. The format specification is given in the Date Format Specifier table.

+

Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use EXTRACT as a workaround.

@@ -3573,7 +3573,18 @@ NUMERIC.years

Creates an interval of months for NUMERIC years.

- + + + {% highlight java %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + + +

Creates an interval of months for NUMERIC quarters.

+

E.g., 2.quarters returns 6.

+ + {% highlight java %} @@ -3586,6 +3597,19 @@ NUMERIC.months + + + {% highlight java %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + + +

Creates an interval of milliseconds for NUMERIC weeks.

+

E.g., 2.weeks returns 1209600000.

+ + + {% highlight java %} @@ -3756,8 +3780,7 @@ dateFormat(TIMESTAMP, STRING) {% endhighlight %} -

Returns a string that formats TIMESTAMP with a specified format STRING. The format specification is given in the Date Format Specifier table.

-

E.g., dateFormat(ts, '%Y, %d %M') results in strings formatted as "2017, 05 May".

+

Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.

@@ -3832,6 +3855,19 @@ NUMERIC.years + + + {% highlight scala %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + + +

Creates an interval of months for NUMERIC quarters.

+

E.g., 2.quarters returns 6.

+ + + {% highlight scala %} @@ -3844,6 +3880,19 @@ NUMERIC.months + + + {% highlight scala %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + + +

Creates an interval of milliseconds for NUMERIC weeks.

+

E.g., 2.weeks returns 1209600000.

+ + + {% highlight scala %} @@ -4014,8 +4063,7 @@ dateFormat(TIMESTAMP, STRING) {% endhighlight %} -

Returns a string that formats TIMESTAMP with a specified format STRING. The format specification is given in the Date Format Specifier table.

-

E.g., dateFormat('ts, "%Y, %d %M") results in strings formatted as "2017, 05 May".

+

Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.

@@ -4186,7 +4234,7 @@ CAST(value AS type)

Returns a new value being cast to type type. See the supported types here.

-

E.g., CAST('42' AS INT) returns 42.

+

E.g., CAST('42' AS INT) returns 42; CAST(NULL AS VARCHAR) returns NULL of type VARCHAR.

@@ -4211,7 +4259,7 @@ ANY.cast(TYPE)

Returns a new ANY being cast to type TYPE. See the supported types here.

-

E.g., '42'.cast(INT) returns 42.

+

E.g., '42'.cast(INT) returns 42; Null(STRING) returns NULL of type STRING.

@@ -4236,7 +4284,7 @@ ANY.cast(TYPE)

Returns a new ANY being cast to type TYPE. See the supported types here.

-

E.g., "42".cast(Types.INT) returns 42.

+

E.g., "42".cast(Types.INT) returns 42; Null(Types.STRING) returns NULL of type STRING.

diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql.md index b1bd572d4d2f6..77eddb8053c87 100644 --- a/docs/dev/table/sql.md +++ b/docs/dev/table/sql.md @@ -22,7 +22,7 @@ specific language governing permissions and limitations under the License. --> -SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can seamlessly mixed and are holistically optimized and translated into a single program. +SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program. In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources. @@ -163,6 +163,7 @@ joinCondition: tableReference: tablePrimary + [ matchRecognize ] [ [ AS ] alias [ '(' columnAlias [, columnAlias ]* ')' ] ] tablePrimary: @@ -196,6 +197,45 @@ windowSpec: ] ')' +matchRecognize: + MATCH_RECOGNIZE '(' + [ PARTITION BY expression [, expression ]* ] + [ ORDER BY orderItem [, orderItem ]* ] + [ MEASURES measureColumn [, measureColumn ]* ] + [ ONE ROW PER MATCH ] + [ AFTER MATCH + ( SKIP TO NEXT ROW + | SKIP PAST LAST ROW + | SKIP TO FIRST variable + | SKIP TO LAST variable + | SKIP TO variable ) + ] + PATTERN '(' pattern ')' + DEFINE variable AS condition [, variable AS condition ]* + ')' + +measureColumn: + expression AS alias + +pattern: + patternTerm [ '|' patternTerm ]* + +patternTerm: + patternFactor [ patternFactor ]* + +patternFactor: + variable [ patternQuantifier ] + +patternQuantifier: + '*' + | '*?' + | '+' + | '+?' + | '?' + | '??' + | '{' { [ minRepeat ], [ maxRepeat ] } '}' ['?'] + | '{' repeat '}' + {% endhighlight %} Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java: @@ -756,7 +796,6 @@ Group windows are defined in the `GROUP BY` clause of a SQL query. Just like que - #### Time Attributes For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes](streaming/time_attributes.html) to learn how to define time attributes. @@ -902,6 +941,52 @@ val result4 = tableEnv.sqlQuery( {% top %} +### Pattern Recognition + +
+ + + + + + + + + + + + + + +
OperationDescription
+ MATCH_RECOGNIZE
+ Streaming + 		+

Searches for a given pattern in a streaming table according to the MATCH_RECOGNIZE ISO standard. This makes it possible to express complex event processing (CEP) logic in SQL queries.

+

For a more detailed description, see the dedicated page for detecting patterns in tables.

+ +{% highlight sql %} +SELECT T.aid, T.bid, T.cid +FROM MyTable +MATCH_RECOGNIZE ( + PARTITION BY userid + ORDER BY proctime + MEASURES + A.id AS aid, + B.id AS bid, + C.id AS cid + PATTERN (A B C) + DEFINE + A AS name = 'a', + B AS name = 'b', + C AS name = 'c' +) AS T +{% endhighlight %} +
+
+ +{% top %} + Data Types ---------- diff --git a/docs/dev/table/streaming/dynamic_tables.md b/docs/dev/table/streaming/dynamic_tables.md index 698cd673185eb..f8bcb9405c61b 100644 --- a/docs/dev/table/streaming/dynamic_tables.md +++ b/docs/dev/table/streaming/dynamic_tables.md @@ -53,12 +53,12 @@ The following table compares traditional relational algebra and stream processin -Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update materialized views and updates a materialized view as soon as its base tables are updated. +Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update a materialized view as soon as its base tables are updated. The connection between eager view maintenance and SQL queries on streams becomes obvious if we consider the following: - A database table is the result of a *stream* of `INSERT`, `UPDATE`, and `DELETE` DML statements, often called *changelog stream*. -- A materialized view is defined as a SQL query. In order to update the view, the query is continuously processes the changelog streams of the view's base relations. +- A materialized view is defined as a SQL query. In order to update the view, the query continuously processes the changelog streams of the view's base relations. - The materialized view is the result of the streaming SQL query. With these points in mind, we introduce following concept of *Dynamic tables* in the next section. @@ -66,7 +66,7 @@ With these points in mind, we introduce following concept of *Dynamic tables* in Dynamic Tables & Continuous Queries --------------------------------------- -*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic table are changing over time. They can be queried like static batch tables. Querying a dynamic table yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its input (dynamic) table. Essentially, a continuous query on a dynamic table is very similar to the definition query of a materialized view. +*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic tables are changing over time. They can be queried like static batch tables. Querying dynamic tables yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its (dynamic) input tables. Essentially, a continuous query on a dynamic table is very similar to a query that defines a materialized view. It is important to note that the result of a continuous query is always semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables. @@ -177,7 +177,7 @@ When converting a dynamic table into a stream or writing it to an external syste

-* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete message*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a dynamic table by encoding `INSERT` and `UPDATE` changes as upsert message and `DELETE` changes as delete message. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream. +* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete messages*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a stream by encoding `INSERT` and `UPDATE` changes as upsert messages and `DELETE` changes as delete messages. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream.
Dynamic tables diff --git a/docs/dev/table/streaming/joins.md b/docs/dev/table/streaming/joins.md index 24c33b9be6306..508e8c70221c3 100644 --- a/docs/dev/table/streaming/joins.md +++ b/docs/dev/table/streaming/joins.md @@ -136,14 +136,14 @@ SELECT o.amount * r.rate AS amount FROM Orders AS o, - LATERAL TABLE (Rates(o.proctime)) AS r + LATERAL TABLE (Rates(o.rowtime)) AS r WHERE r.currency = o.currency {% endhighlight %} -Each record from the probe side will be joined with the version of the build side table at the time of the correlated time attribute of the probe side record. +Each record from the probe side will be joined with the version of the build side table at the time of the correlated time attribute of the probe side record. In order to support updates (overwrites) of previous values on the build side table, the table must define a primary key. -In our example, each record from `Orders` will be joined with the version of `Rates` at time `o.proctime`. Because the time attribute defines a processing-time notion, a newly appended order is always joined with the most recent version of `Rates` when executing the operation. The `currency` field has been defined as the primary key of `Rates` before and is used to connect both tables in our example. +In our example, each record from `Orders` will be joined with the version of `Rates` at time `o.rowtime`. The `currency` field has been defined as the primary key of `Rates` before and is used to connect both tables in our example. If the query were using a processing-time notion, a newly appended order would always be joined with the most recent version of `Rates` when executing the operation. In contrast to [regular joins](#regular-joins), this means that if there is a new record on the build side, it will not affect the previous results of the join. This again allows Flink to limit the number of elements that must be kept in the state. @@ -189,14 +189,35 @@ val result = orders
+**Note**: State retention defined in a [query configuration](query_configuration.html) is not yet implemented for temporal joins. +This means that the required state to compute the query result might grow infinitely depending on the number of distinct primary keys for the history table. + ### Processing-time Temporal Joins With a processing-time time attribute, it is impossible to pass _past_ time attributes as an argument to the temporal table function. -By definition, it is always the current timestamp. Thus, processing-time temporal table function invocations will always return the latest known versions of the underlying table +By definition, it is always the current timestamp. Thus, invocations of a processing-time temporal table function will always return the latest known versions of the underlying table and any updates in the underlying history table will also immediately overwrite the current values. -Only the latest versions (with respect to the defined primary key) of the build side records are kept in the state. New updates will have no effect on the previously results emitted/processed records from the probe side. +Only the latest versions (with respect to the defined primary key) of the build side records are kept in the state. +Updates of the build side will have no effect on previously emitted join results. -One can think about processing-time temporal join as a simple `HashMap` that stores all of the records from the build side. +One can think about a processing-time temporal join as a simple `HashMap` that stores all of the records from the build side. When a new record from the build side has the same key as some previous record, the old value is just simply overwritten. Every record from the probe side is always evaluated against the most recent/current state of the `HashMap`. + +### Event-time Temporal Joins + +With an event-time time attribute (i.e., a rowtime attribute), it is possible to pass _past_ time attributes to the temporal table function. +This allows for joining the two tables at a common point in time. + +Compared to processing-time temporal joins, the temporal table does not only keep the latest version (with respect to the defined primary key) of the build side records in the state +but stores all versions (identified by time) since the last watermark. + +For example, an incoming row with an event-time timestamp of `12:30:00` that is appended to the probe side table +is joined with the version of the build side table at time `12:30:00` according to the [concept of temporal tables](temporal_tables.html). +Thus, the incoming row is only joined with rows that have a timestamp lower or equal to `12:30:00` with +applied updates according to the primary key until this point in time. + +By definition of event time, [watermarks]({{ site.baseurl }}/dev/event_time.html) allow the join operation to move +forward in time and discard versions of the build table that are no longer necessary because no incoming row with +lower or equal timestamp is expected. diff --git a/docs/dev/table/streaming/match_recognize.md b/docs/dev/table/streaming/match_recognize.md new file mode 100644 index 0000000000000..3c805cf013333 --- /dev/null +++ b/docs/dev/table/streaming/match_recognize.md @@ -0,0 +1,843 @@ +--- +title: 'Detecting Patterns in Tables' +nav-parent_id: streaming_tableapi +nav-title: 'Detecting Patterns' +nav-pos: 5 +is_beta: true +--- + + +It is a common use case to search for a set of event patterns, especially in case of data streams. Flink +comes with a [complex event processing (CEP) library]({{ site.baseurl }}/dev/libs/cep.html) which allows for pattern detection in event streams. Furthermore, Flink's +SQL API provides a relational way of expressing queries with a large set of built-in functions and rule-based optimizations that can be used out of the box. + +In December 2016, the International Organization for Standardization (ISO) released a new version of the SQL standard which includes _Row Pattern Recognition in SQL_ ([ISO/IEC TR 19075-5:2016](https://standards.iso.org/ittf/PubliclyAvailableStandards/c065143_ISO_IEC_TR_19075-5_2016.zip)). It allows Flink to consolidate CEP and SQL API using the `MATCH_RECOGNIZE` clause for complex event processing in SQL. + +A `MATCH_RECOGNIZE` clause enables the following tasks: +* Logically partition and order the data that is used with the `PARTITION BY` and `ORDER BY` clauses. +* Define patterns of rows to seek using the `PATTERN` clause. These patterns use a syntax similar to that of regular expressions. +* The logical components of the row pattern variables are specified in the `DEFINE` clause. +* Define measures, which are expressions usable in other parts of the SQL query, in the `MEASURES` clause. + +The following example illustrates the syntax for basic pattern recognition: + +{% highlight sql %} +SELECT T.aid, T.bid, T.cid +FROM MyTable +MATCH_RECOGNIZE ( + PARTITION BY userid + ORDER BY proctime + MEASURES + A.id AS aid, + B.id AS bid, + C.id AS cid + PATTERN (A B C) + DEFINE + A AS name = 'a', + B AS name = 'b', + C AS name = 'c' +) AS T +{% endhighlight %} + +This page will explain each keyword in more detail and will illustrate more complex examples. + +Attention Flink's implementation of the `MATCH_RECOGNIZE` clause is a subset of the full standard. Only those features documented in the following sections are supported. Since the development is still in an early phase, please also take a look at the [known limitations](#known-limitations). + +* This will be replaced by the TOC +{:toc} + +Introduction and Examples +------------------------- + +### Installation Guide + +The pattern recognition feature uses the Apache Flink's CEP library internally. In order to be able to use the `MATCH_RECOGNIZE` clause, +the library needs to be added as a dependency to your Maven project. + +{% highlight xml %} + + org.apache.flink + flink-cep{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + +Alternatively, you can also add the dependency to the cluster classpath (see the [dependency section]({{ site.baseurl}}/projectsetup/dependencies.html) for more information). + +If you want to use the `MATCH_RECOGNIZE` clause in the [SQL Client]({{ site.baseurl}}/dev/table/sqlClient.html), +you don't have to do anything as all the dependencies are included by default. + +### SQL Semantics + +Every `MATCH_RECOGNIZE` query consists of the following clauses: + +* [PARTITION BY](#partitioning) - defines the logical partitioning of the table; similar to a `GROUP BY` operation. +* [ORDER BY](#order-of-events) - specifies how the incoming rows should be ordered; this is essential as patterns depend on an order. +* [MEASURES](#define--measures) - defines output of the clause; similar to a `SELECT` clause. +* [ONE ROW PER MATCH](#output-mode) - output mode which defines how many rows per match should be produced. +* [AFTER MATCH SKIP](#after-match-strategy) - specifies where the next match should start; this is also a way to control how many distinct matches a single event can belong to. +* [PATTERN](#defining-pattern) - allows constructing patterns that will be searched for using a _regular expression_-like syntax. +* [DEFINE](#define--measures) - this section defines the conditions that the pattern variables must satisfy. + +Attention Currently, the `MATCH_RECOGNIZE` clause can only be applied to an [append table](dynamic_tables.html#update-and-append-queries). Furthermore, it always produces +an append table as well. + +### Examples + +For our examples, we assume that a table `Ticker` has been registered. The table contains prices of stocks at a particular point in time. + +The table has a following schema: + +{% highlight text %} +Ticker + |-- symbol: String # symbol of the stock + |-- price: Long # price of the stock + |-- tax: Long # tax liability of the stock + |-- rowtime: TimeIndicatorTypeInfo(rowtime) # point in time when the change to those values happened +{% endhighlight %} + +For simplification, we only consider the incoming data for a single stock `ACME`. A ticker could look similar to the following table where rows are continuously appended. + +{% highlight text %} +symbol rowtime price tax +====== ==================== ======= ======= +'ACME' '01-Apr-11 10:00:00' 12 1 +'ACME' '01-Apr-11 10:00:01' 17 2 +'ACME' '01-Apr-11 10:00:02' 19 1 +'ACME' '01-Apr-11 10:00:03' 21 3 +'ACME' '01-Apr-11 10:00:04' 25 2 +'ACME' '01-Apr-11 10:00:05' 18 1 +'ACME' '01-Apr-11 10:00:06' 15 1 +'ACME' '01-Apr-11 10:00:07' 14 2 +'ACME' '01-Apr-11 10:00:08' 24 2 +'ACME' '01-Apr-11 10:00:09' 25 2 +'ACME' '01-Apr-11 10:00:10' 19 1 +{% endhighlight %} + +The task is now to find periods of a constantly decreasing price of a single ticker. For this, one could write a query like: + +{% highlight sql %} +SELECT * +FROM Ticker +MATCH_RECOGNIZE ( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + START_ROW.rowtime AS start_tstamp, + LAST(PRICE_DOWN.rowtime) AS bottom_tstamp, + LAST(PRICE_UP.rowtime) AS end_tstamp + ONE ROW PER MATCH + AFTER MATCH SKIP TO LAST PRICE_UP + PATTERN (START_ROW PRICE_DOWN+ PRICE_UP) + DEFINE + PRICE_DOWN AS + (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR + PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1), + PRICE_UP AS + PRICE_UP.price > LAST(PRICE_DOWN.price, 1) + ) MR; +{% endhighlight %} + +The query partitions the `Ticker` table by the `symbol` column and orders it by the `rowtime` time attribute. + +The `PATTERN` clause specifies that we are interested in a pattern with a starting event `START_ROW` that is followed by one or more `PRICE_DOWN` events and concluded with a `PRICE_UP` event. If such a pattern can be found, the next pattern match will be seeked at the last `PRICE_UP` event as indicated by the `AFTER MATCH SKIP TO LAST` clause. + +The `DEFINE` clause specifies the conditions that need to be met for a `PRICE_DOWN` and `PRICE_UP` event. Although the `START_ROW` pattern variable is not present it has an implicit condition that is evaluated always as `TRUE`. + +A pattern variable `PRICE_DOWN` is defined as a row with a price that is smaller than the price of the last row that met the `PRICE_DOWN` condition. For the initial case or when there is no last row that met the `PRICE_DOWN` condition, the price of the row should be smaller than the price of the preceding row in the pattern (referenced by `START_ROW`). + +A pattern variable `PRICE_UP` is defined as a row with a price that is larger than the price of the last row that met the `PRICE_DOWN` condition. + +This query produces a summary row for each period in which the price of a stock was continuously decreasing. + +The exact representation of the output rows is defined in the `MEASURES` part of the query. The number of output rows is defined by the `ONE ROW PER MATCH` output mode. + +{% highlight text %} + symbol start_tstamp bottom_tstamp end_tstamp +========= ================== ================== ================== +ACME 01-APR-11 10:00:04 01-APR-11 10:00:07 01-APR-11 10:00:08 +{% endhighlight %} + +The resulting row describes a period of falling prices that started at `01-APR-11 10:00:04` and +achieved the lowest price at `01-APR-11 10:00:07` that increased again at `01-APR-11 10:00:08`. + +Partitioning +------------ + +It is possible to look for patterns in partitioned data, e.g., trends for a single ticker or a particular user. This can be expressed using the `PARTITION BY` clause. The clause is similar to using `GROUP BY` for aggregations. + +Attention It is highly advised to partition the incoming data because otherwise the `MATCH_RECOGNIZE` clause will be translated +into a non-parallel operator to ensure global ordering. + +Order of Events +--------------- + +Apache Flink allows for searching for patterns based on time; either [processing time or event time](time_attributes.html). + +In case of event time, the events are sorted before they are passed to the internal pattern state machine. As a consequence, the +produced output will be correct regardless of the order in which rows are appended to the table. Instead, the pattern is evaluated in the order specified by the time contained in each row. + +The `MATCH_RECOGNIZE` clause assumes a [time attribute](time_attributes.html) with ascending ordering as the first argument to `ORDER BY` clause. + +For the example `Ticker` table, a definition like `ORDER BY rowtime ASC, price DESC` is valid but `ORDER BY price, rowtime` or `ORDER BY rowtime DESC, price ASC` is not. + +Define & Measures +----------------- + +The `DEFINE` and `MEASURES` keywords have similar meanings to the `WHERE` and `SELECT` clauses in a simple SQL query. + +The `MEASURES` clause defines what will be included in the output of a matching pattern. It can project columns and define expressions for evaluation. +The number of produced rows depends on the [output mode](#output-mode) setting. + +The `DEFINE` clause specifies conditions that rows have to fulfill in order to be classified to a corresponding [pattern variable](#defining-pattern). +If a condition is not defined for a pattern variable, a default condition will be used which evaluates to `true` for every row. + +For a more detailed explanation about expressions that can be used in those clauses, please have a look at the [event stream navigation](#pattern-navigation) section. + +Defining a Pattern +------------------ + +The `MATCH_RECOGNIZE` clause allows users to search for patterns in event streams using a powerful and expressive syntax +that is somewhat similar to the widespread regular expression syntax. + +Every pattern is constructed from basic building blocks, called _pattern variables_, to which operators (quantifiers and other modifiers) can be applied. The whole pattern must be enclosed in +brackets. + +An example pattern could look like: + +{% highlight sql %} +PATTERN (A B+ C* D) +{% endhighlight %} + +One may use the following operators: + +* _Concatenation_ - a pattern like `(A B)` means that the contiguity is strict between `A` and `B`. Therefore, there can be no rows that were not mapped to `A` or `B` in between. +* _Quantifiers_ - modify the number of rows that can be mapped to the pattern variable. + * `*` — _0_ or more rows + * `+` — _1_ or more rows + * `?` — _0_ or _1_ rows + * `{ n }` — exactly _n_ rows (_n > 0_) + * `{ n, }` — _n_ or more rows (_n ≥ 0_) + * `{ n, m }` — between _n_ and _m_ (inclusive) rows (_0 ≤ n ≤ m, 0 < m_) + * `{ , m }` — between _0_ and _m_ (inclusive) rows (_m > 0_) + +Attention Patterns that can potentially produce an empty match are not supported. +Examples of such patterns are `PATTERN (A*)`, `PATTERN (A? B*)`, `PATTERN (A{0,} B{0,} C*)`, etc. + +### Greedy & Reluctant Quantifiers + +Each quantifier can be either _greedy_ (default behavior) or _reluctant_. Greedy quantifiers try to match +as many rows as possible while reluctant quantifiers try to match as few as possible. + +In order to illustrate the difference, one can view the following example with a query where a greedy quantifier is applied to the `B` variable: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + C.price AS lastPrice + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A B* C) + DEFINE + A AS A.price > 10, + B AS B.price < 15, + C AS C.price > 12 + ) +{% endhighlight %} + +Given we have the following input: + +{% highlight text %} + symbol tax price rowtime +======= ===== ======== ===================== + XYZ 1 10 2018-09-17 10:00:02 + XYZ 2 11 2018-09-17 10:00:03 + XYZ 1 12 2018-09-17 10:00:04 + XYZ 2 13 2018-09-17 10:00:05 + XYZ 1 14 2018-09-17 10:00:06 + XYZ 2 16 2018-09-17 10:00:07 +{% endhighlight %} + +The pattern above will produce the following output: + +{% highlight text %} + symbol lastPrice +======== =========== + XYZ 16 +{% endhighlight %} + +The same query where `B*` is modified to `B*?`, which means that `B*` should be reluctant, will produce: + +{% highlight text %} + symbol lastPrice +======== =========== + XYZ 13 + XYZ 16 +{% endhighlight %} + +The pattern variable `B` matches only to the row with price `12` instead of swallowing the rows with prices `12`, `13`, and `14`. + +Attention It is not possible to use a greedy quantifier for the last +variable of a pattern. Thus, a pattern like `(A B*)` is not allowed. This can be easily worked around by introducing an artificial state +(e.g. `C`) that has a negated condition of `B`. So you could use a query like: + +{% highlight sql %} +PATTERN (A B* C) +DEFINE + A AS condA(), + B AS condB(), + C AS NOT condB() +{% endhighlight %} + +Attention The optional reluctant quantifier (`A??` or `A{0,1}?`) is not supported right now. + +Output Mode +----------- + +The _output mode_ describes how many rows should be emitted for every found match. The SQL standard describes two modes: +- `ALL ROWS PER MATCH` +- `ONE ROW PER MATCH`. + +Currently, the only supported output mode is `ONE ROW PER MATCH` that will always produce one output summary row for each found match. + +The schema of the output row will be a concatenation of `[partitioning columns] + [measures columns]` in that particular order. + +The following example shows the output of a query defined as: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + FIRST(A.price) AS startPrice, + LAST(A.price) AS topPrice, + B.price AS lastPrice + ONE ROW PER MATCH + PATTERN (A+ B) + DEFINE + A AS LAST(A.price, 1) IS NULL OR A.price > LAST(A.price, 1), + B AS B.price < LAST(A.price) + ) +{% endhighlight %} + +For the following input rows: + +{% highlight text %} + symbol tax price rowtime +======== ===== ======== ===================== + XYZ 1 10 2018-09-17 10:00:02 + XYZ 2 12 2018-09-17 10:00:03 + XYZ 1 13 2018-09-17 10:00:04 + XYZ 2 11 2018-09-17 10:00:05 +{% endhighlight %} + +The query will produce the following output: + +{% highlight text %} + symbol startPrice topPrice lastPrice +======== ============ ========== =========== + XYZ 10 13 11 +{% endhighlight %} + +The pattern recognition is partitioned by the `symbol` column. Even though not explicitly mentioned in the `MEASURES` clause, the partitioned column is added at the beginning of the result. + +Pattern Navigation +------------------ + +The `DEFINE` and `MEASURES` clauses allow for navigating within the list of rows that (potentially) match a pattern. + +This section discusses this navigation for declaring conditions or producing output results. + +### Pattern Variable Referencing + +A _pattern variable reference_ allows a set of rows mapped to a particular pattern variable in the `DEFINE` or `MEASURES` clauses to be referenced. + +For example, the expression `A.price` describes a set of rows mapped so far to `A` plus the current row +if we try to match the current row to `A`. If an expression in the `DEFINE`/`MEASURES` clause requires a single row (e.g. `A.price` or `A.price > 10`), +it selects the last value belonging to the corresponding set. + +If no pattern variable is specified (e.g. `SUM(price)`), an expression references the default pattern variable `*` which references all variables in the pattern. +In other words, it creates a list of all the rows mapped so far to any variable plus the current row. + +#### Example + +For a more thorough example, one can take a look at the following pattern and corresponding conditions: + +{% highlight sql %} +PATTERN (A B+) +DEFINE + A AS A.price > 10, + B AS B.price > A.price AND SUM(price) < 100 AND SUM(B.price) < 80 +{% endhighlight %} + +The following table describes how those conditions are evaluated for each incoming event. + +The table consists of the following columns: + * `#` - the row identifier that uniquely identifies an incoming row in the lists `[A.price]`/`[B.price]`/`[price]`. + * `price` - the price of the incoming row. + * `[A.price]`/`[B.price]`/`[price]` - describe lists of rows which are used in the `DEFINE` clause to evaluate conditions. + * `Classifier` - the classifier of the current row which indicates the pattern variable the row is mapped to. + * `A.price`/`B.price`/`SUM(price)`/`SUM(B.price)` - describes the result after those expressions have been evaluated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#priceClassifier[A.price][B.price][price]A.priceB.priceSUM(price)SUM(B.price)
#110-> A#1--10---
#215-> B#1#2#1, #210152515
#320-> B#1#2, #3#1, #2, #310204535
#431-> B#1#2, #3, #4#1, #2, #3, #410317666
#535#1#2, #3, #4, #5#1, #2, #3, #4, #51035111101
+ +As can be seen in the table, the first row is mapped to pattern variable `A` and subsequent rows are mapped to pattern variable `B`. However, the last row does not fulfill the `B` condition because the sum over all mapped rows `SUM(price)` and the sum over all rows in `B` exceed the specified thresholds. + +Attention Please note that aggregations such as `SUM` are not supported yet. They are only used for explanation here. + +### Logical Offsets + +_Logical offsets_ enable navigation within the events that were mapped to a particular pattern variable. This can be expressed +with two corresponding functions: + + + + + + + + + + + + + + + + + + +
Offset functionsDescription
+ {% highlight text %} +LAST(variable.field, n) +{% endhighlight %} + +

Returns the value of the field from the event that was mapped to the n-th last element of the variable. The counting starts at the last element mapped.

+
+ {% highlight text %} +FIRST(variable.field, n) +{% endhighlight %} + +

Returns the value of the field from the event that was mapped to the n-th element of the variable. The counting starts at the first element mapped.

+
+ +#### Examples + +For a more thorough example, one can take a look at the following pattern and corresponding conditions: + +{% highlight sql %} +PATTERN (A B+) +DEFINE + A AS A.price > 10, + B AS (LAST(B.price, 1) IS NULL OR B.price > LAST(B.price, 1)) AND + (LAST(B.price, 2) IS NULL OR B.price > 2 * LAST(B.price, 2)) +{% endhighlight %} + +The following table describes how those conditions are evaluated for each incoming event. + +The table consists of the following columns: + * `price` - the price of the incoming row. + * `Classifier` - the classifier of the current row which indicates the pattern variable the row is mapped to. + * `LAST(B.price, 1)`/`LAST(B.price, 2)` - describes the result after those expressions have been evaluated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
priceClassifierLAST(B.price, 1)LAST(B.price, 2)Comment
10-> A
15-> BnullnullNotice that LAST(A.price, 1) is null because there is still nothing mapped to B.
20-> B15null
31-> B2015
353120Not mapped because 35 < 2 * 20.
+ +It might also make sense to use the default pattern variable with logical offsets. + +In this case, an offset considers all the rows mapped so far: + +{% highlight sql %} +PATTERN (A B? C) +DEFINE + B AS B.price < 20, + C AS LAST(price, 1) < C.price +{% endhighlight %} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
priceClassifierLAST(price, 1)Comment
10-> A
15-> B
20-> C15LAST(price, 1) is evaluated as the price of the row mapped to the B variable.
+ +If the second row did not map to the `B` variable, we would have the following results: + + + + + + + + + + + + + + + + + + + + + + + + +
priceClassifierLAST(price, 1)Comment
10-> A
20-> C10LAST(price, 1) is evaluated as the price of the row mapped to the A variable.
+ +It is also possible to use multiple pattern variable references in the first argument of the `FIRST/LAST` functions. This way, one can write an expression that accesses multiple columns. +However, all of them must use the same pattern variable. In other words, the value of the `LAST`/`FIRST` function must be computed in a single row. + +Thus, it is possible to use `LAST(A.price * A.tax)`, but an expression like `LAST(A.price * B.tax)` is not allowed. + +After Match Strategy +-------------------- + +The `AFTER MATCH SKIP` clause specifies where to start a new matching procedure after a complete match was found. + +There are four different strategies: +* `SKIP PAST LAST ROW` - resumes the pattern matching at the next row after the last row of the current match. +* `SKIP TO NEXT ROW` - continues searching for a new match starting at the next row after the starting row of the match. +* `SKIP TO LAST variable` - resumes the pattern matching at the last row that is mapped to the specified pattern variable. +* `SKIP TO FIRST variable` - resumes the pattern matching at the first row that is mapped to the specified pattern variable. + +This is also a way to specify how many matches a single event can belong to. For example, with the `SKIP PAST LAST ROW` strategy every event can belong to at most one match. + +#### Examples + +In order to better understand the differences between those strategies one can take a look at the following example. + +For the following input rows: + +{% highlight text %} + symbol tax price rowtime +======== ===== ======= ===================== + XYZ 1 7 2018-09-17 10:00:01 + XYZ 2 9 2018-09-17 10:00:02 + XYZ 1 10 2018-09-17 10:00:03 + XYZ 2 5 2018-09-17 10:00:04 + XYZ 2 17 2018-09-17 10:00:05 + XYZ 2 14 2018-09-17 10:00:06 +{% endhighlight %} + +We evaluate the following query with different strategies: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + SUM(A.price) AS sumPrice, + FIRST(rowtime) AS startTime, + LAST(rowtime) AS endTime + ONE ROW PER MATCH + [AFTER MATCH STRATEGY] + PATTERN (A+ C) + DEFINE + A AS SUM(A.price) < 30 + ) +{% endhighlight %} + +The query returns the sum of the prices of all rows mapped to `A` and the first and last timestamp of the overall match. + +Attention Please note that aggregations such as `SUM` are not supported yet. They are only used for explanation here. + +The query will produce different results based on which `AFTER MATCH` strategy was used: + +##### `AFTER MATCH SKIP PAST LAST ROW` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +The first result matched against the rows #1, #2, #3, #4. + +The second result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO NEXT ROW` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 24 2018-09-17 10:00:02 2018-09-17 10:00:05 + XYZ 15 2018-09-17 10:00:03 2018-09-17 10:00:05 + XYZ 22 2018-09-17 10:00:04 2018-09-17 10:00:06 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +Again, the first result matched against the rows #1, #2, #3, #4. + +Compared to the previous strategy, the next match includes row #2 again for the next matching. Therefore, the second result matched against the rows #2, #3, #4, #5. + +The third result matched against the rows #3, #4, #5. + +The forth result matched against the rows #4, #5, #6. + +The last result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO LAST A` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 15 2018-09-17 10:00:03 2018-09-17 10:00:05 + XYZ 22 2018-09-17 10:00:04 2018-09-17 10:00:06 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +Again, the first result matched against the rows #1, #2, #3, #4. + +Compared to the previous strategy, the next match includes only row #3 (mapped to `A`) again for the next matching. Therefore, the second result matched against the rows #3, #4, #5. + +The third result matched against the rows #4, #5, #6. + +The last result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO FIRST A` + +This combination will produce a runtime exception because one would always try to start a new match where the +last one started. This would produce an infinite loop and, thus, is prohibited. + +One has to keep in mind that in case of the `SKIP TO FIRST/LAST variable` strategy it might be possible that there are no rows mapped to that +variable (e.g. for pattern `A*`). In such cases, a runtime exception will be thrown as the standard requires a valid row to continue the +matching. + + +### Controlling Memory Consumption + +Memory consumption is an important consideration when writing `MATCH_RECOGNIZE` queries, as the space of potential matches is built in a breadth-first-like manner. +Having that in mind, one must make sure that the pattern can finish. Preferably with a reasonable number of rows mapped to the match as they have to fit into memory. + +For example, the pattern must not have a quantifier without an upper limit that accepts every single row. Such a pattern could look like this: + +{% highlight sql %} +PATTERN (A B+ C) +DEFINE + A as A.price > 10, + C as C.price > 20 +{% endhighlight %} + +The query will map every incoming row to the `B` variable and thus will never finish. This query could be fixed, e.g., by negating the condition for `C`: + +{% highlight sql %} +PATTERN (A B+ C) +DEFINE + A as A.price > 10, + B as B.price <= 20, + C as C.price > 20 +{% endhighlight %} + +Or by using the [reluctant quantifier](#greedy--reluctant-quantifiers): + +{% highlight sql %} +PATTERN (A B+? C) +DEFINE + A as A.price > 10, + C as C.price > 20 +{% endhighlight %} + +Attention Please note that the `MATCH_RECOGNIZE` clause does not use a configured [state retention time](query_configuration.html#idle-state-retention-time). As of now, there is also no possibility to define a time restriction on the pattern to finish because there is no such possibility in the SQL standard. The community is in the process of designing a proper syntax for that +feature right now. + +Known Limitations +----------------- + +Flink's implementation of the `MATCH_RECOGNIZE` clause is an ongoing effort, and some features of the SQL standard are not yet supported. + +Unsupported features include: +* Pattern expressions: + * Pattern groups - this means that e.g. quantifiers can not be applied to a subsequence of the pattern. Thus, `(A (B C)+)` is not a valid pattern. + * Alterations - patterns like `PATTERN((A B | C D) E)`, which means that either a subsequence `A B` or `C D` has to be found before looking for the `E` row. + * `PERMUTE` operator - which is equivalent to all permutations of variables that it was applied to e.g. `PATTERN (PERMUTE (A, B, C))` = `PATTERN (A B C | A C B | B A C | B C A | C A B | C B A)`. + * Anchors - `^, $`, which denote beginning/end of a partition, those do not make sense in the streaming context and will not be supported. + * Exclusion - `PATTERN ({- A -} B)` meaning that `A` will be looked for but will not participate in the output. This works only for the `ALL ROWS PER MATCH` mode. + * Reluctant optional quantifier - `PATTERN A??` only the greedy optional quantifier is supported. +* `ALL ROWS PER MATCH` output mode - which produces an output row for every row that participated in the creation of a found match. This also means: + * that the only supported semantic for the `MEASURES` clause is `FINAL` + * `CLASSIFIER` function, which returns the pattern variable that a row was mapped to, is not yet supported. +* `SUBSET` - which allows creating logical groups of pattern variables and using those groups in the `DEFINE` and `MEASURES` clauses. +* Physical offsets - `PREV/NEXT`, which indexes all events seen rather than only those that were mapped to a pattern variable(as in [logical offsets](#logical-offsets) case). +* Extracting time attributes - there is currently no possibility to get a time attribute for subsequent time-based operations. +* Aggregates - one cannot use aggregates in `MEASURES` nor `DEFINE` clauses. +* User defined functions cannot be used within `MATCH_RECOGNIZE`. +* `MATCH_RECOGNIZE` is supported only for SQL. There is no equivalent in the Table API. diff --git a/docs/dev/table/streaming/query_configuration.md b/docs/dev/table/streaming/query_configuration.md index dc6d11ff6ec17..acfe8e845f1d2 100644 --- a/docs/dev/table/streaming/query_configuration.md +++ b/docs/dev/table/streaming/query_configuration.md @@ -1,7 +1,7 @@ --- title: "Query Configuration" nav-parent_id: streaming_tableapi -nav-pos: 5 +nav-pos: 6 --- -This page provides details on setting up and configuring distributed file systems for use with Flink. +This page provides details on setting up and configuring different file systems for use with Flink. +We start by describing how to use and configure the different file systems that are supported by Flink +out-of-the-box, before describing the necessary steps in order to add support about other/custom file system +implementations. ## Flink's File System support -Flink uses file systems both as a source and sink in streaming/batch applications, and as a target for checkpointing. +Flink uses file systems both as *sources* and *sinks* in streaming/batch applications and as a target for *checkpointing*. These file systems can for example be *Unix/Windows file systems*, *HDFS*, or even object stores like *S3*. The file system used for a specific file is determined by the file URI's scheme. For example `file:///home/user/text.txt` refers to a file in the local file system, while `hdfs://namenode:50010/data/user/text.txt` refers to a file in a specific HDFS cluster. File systems are represented via the `org.apache.flink.core.fs.FileSystem` class, which captures the ways to access and modify -files and objects in that file system. FileSystem instances are instantiates once per process and then cached / pooled, to -avoid configuration overhead per stream creation, and to enforce certain constraints, like connection/stream limits. +files and objects in that file system. FileSystem instances are instantiated once per process and then cached / pooled, to +avoid configuration overhead per stream creation and to enforce certain constraints, such as connection/stream limits. ### Built-in File Systems -Flink directly implements the following file systems: +Flink ships with support for most of the popular file systems, namely *local*, *hadoop-compatible*, *S3*, *MapR FS* +and *OpenStack Swift FS*. Each of these is identified by the scheme included in the URI of the provide file path. + +Flink ships with implementations for the following file systems: - **local**: This file system is used when the scheme is *"file://"*, and it represents the file system of the local machine, including any NFS or SAN that is mounted into that local file system. - **S3**: Flink directly provides file systems to talk to Amazon S3. There are two alternative implementations, `flink-s3-fs-presto` - and `flink-s3-fs-hadoop`. Both implementations are self-contained with no dependency footprint, there is no need to add Hadoop to + and `flink-s3-fs-hadoop`. Both implementations are self-contained with no dependency footprint. There is no need to add Hadoop to the classpath to use them. Both internally use some Hadoop code, but "shade away" all classes to avoid any dependency conflicts. - - `flink-s3-fs-presto`, registered under the scheme *"s3://"*, is based on code from the [Presto project](https://prestodb.io/). + - `flink-s3-fs-presto`, registered under the scheme *"s3://"* and *"s3p://"*, is based on code from the [Presto project](https://prestodb.io/). You can configure it the same way you can [configure the Presto file system](https://prestodb.io/docs/0.185/connector/hive.html#amazon-s3-configuration). - `flink-s3-fs-hadoop`, registered under *"s3://"* and *"s3a://"*, based on code from the [Hadoop Project](https://hadoop.apache.org/). @@ -56,7 +62,13 @@ including any NFS or SAN that is mounted into that local file system. To use those file systems when using Flink as a library, add the respective maven dependency (`org.apache.flink:flink-s3-fs-presto:{{ site.version }}` or `org.apache.flink:flink-s3-fs-hadoop:{{ site.version }}`). When starting a Flink application from the Flink binaries, copy or move the respective jar file from the `opt` folder to the `lib` folder. See also [AWS setup](deployment/aws.html) for additional details. - + + Attention: As described above, both Hadoop and Presto "listen" to paths with scheme set to *"s3://"*. This is + convenient for switching between implementations (Hadoop or Presto), but it can lead to non-determinism when both + implementations are required. This can happen when, for example, the job uses the [StreamingFileSink]({{ site.baseurl}}/dev/connectors/streamfile_sink.html) + which only supports Hadoop, but uses Presto for checkpointing. In this case, it is advised to use explicitly *"s3a://"* + as a scheme for the sink (Hadoop) and *"s3p://"* for checkpointing (Presto). + - **MapR FS**: The MapR file system *"maprfs://"* is automatically available when the MapR libraries are in the classpath. - **OpenStack Swift FS**: Flink directly provides a file system to talk to the OpenStack Swift file system, registered under the scheme *"swift://"*. @@ -64,9 +76,9 @@ including any NFS or SAN that is mounted into that local file system. To use it when using Flink as a library, add the respective maven dependency (`org.apache.flink:flink-swift-fs-hadoop:{{ site.version }}` When starting a Flink application from the Flink binaries, copy or move the respective jar file from the `opt` folder to the `lib` folder. -### HDFS and Hadoop File System support +#### HDFS and Hadoop File System support -For all schemes where Flink cannot find a directly supported file system, Flink will try to use Hadoop to instantiate a file system for the respective scheme. +For all schemes where it cannot find a directly supported file system, Flink will try to use Hadoop to instantiate a file system for the respective scheme. All Hadoop file systems are automatically available once `flink-runtime` and the Hadoop libraries are in classpath. That way, Flink seamlessly supports all of Hadoop file systems, and all Hadoop-compatible file systems (HCFS), for example: diff --git a/docs/ops/jobmanager_high_availability.md b/docs/ops/jobmanager_high_availability.md index 2c2f1a6ba19ed..f37115c9e19fa 100644 --- a/docs/ops/jobmanager_high_availability.md +++ b/docs/ops/jobmanager_high_availability.md @@ -133,17 +133,17 @@ Starting zookeeper daemon on host localhost. 
 $ bin/start-cluster.sh
 Starting HA cluster with 2 masters and 1 peers in ZooKeeper quorum.
-Starting jobmanager daemon on host localhost.
-Starting jobmanager daemon on host localhost.
-Starting taskmanager daemon on host localhost.
+Starting standalonesession daemon on host localhost. +Starting standalonesession daemon on host localhost. +Starting taskexecutor daemon on host localhost. 6. **Stop ZooKeeper quorum and cluster**: 
 $ bin/stop-cluster.sh
-Stopping taskmanager daemon (pid: 7647) on localhost.
-Stopping jobmanager daemon (pid: 7495) on host localhost.
-Stopping jobmanager daemon (pid: 7349) on host localhost.
+Stopping taskexecutor daemon (pid: 7647) on localhost.
+Stopping standalonesession daemon (pid: 7495) on host localhost.
+Stopping standalonesession daemon (pid: 7349) on host localhost.
 $ bin/stop-zookeeper-quorum.sh
 Stopping zookeeper daemon (pid: 7101) on host localhost.
diff --git a/docs/ops/security-ssl.md b/docs/ops/security-ssl.md index 4e3716218d206..77099ebf61f45 100644 --- a/docs/ops/security-ssl.md +++ b/docs/ops/security-ssl.md @@ -198,7 +198,7 @@ This example shows how to create a simple keystore / truststore pair. The trusts be shared with other applications. In this example, *myhost.company.org / ip:10.0.2.15* is the node (or service) for the Flink master. {% highlight bash %} -keytool -genkeypair -alias flink.rest -keystore rest.keystore -dname "CN=myhost.company.org" -ext "SAN=dns:myhost.company.org,ip:10.0.2.15" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 +keytool -genkeypair -alias flink.rest -keystore rest.keystore -dname "CN=myhost.company.org" -ext "SAN=dns:myhost.company.org,ip:10.0.2.15" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 -storetype PKCS12 keytool -exportcert -keystore rest.keystore -alias flink.rest -storepass rest_keystore_password -file flink.cer @@ -219,7 +219,7 @@ security.ssl.rest.key-password: rest_key_password Execute the following keytool commands to create a truststore with a self signed CA. {% highlight bash %} -keytool -genkeypair -alias ca -keystore ca.keystore -dname "CN=Sample CA" -storepass ca_keystore_password -keypass ca_key_password -keyalg RSA -keysize 4096 -ext "bc=ca:true" +keytool -genkeypair -alias ca -keystore ca.keystore -dname "CN=Sample CA" -storepass ca_keystore_password -keypass ca_key_password -keyalg RSA -keysize 4096 -ext "bc=ca:true" -storetype PKCS12 keytool -exportcert -keystore ca.keystore -alias ca -storepass ca_keystore_password -file ca.cer @@ -230,7 +230,7 @@ Now create a keystore for the REST endpoint with a certificate signed by the abo Let *flink.company.org / ip:10.0.2.15* be the hostname of the Flink master (JobManager). {% highlight bash %} -keytool -genkeypair -alias flink.rest -keystore rest.signed.keystore -dname "CN=flink.company.org" -ext "SAN=dns:flink.company.org" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 +keytool -genkeypair -alias flink.rest -keystore rest.signed.keystore -dname "CN=flink.company.org" -ext "SAN=dns:flink.company.org" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 -storetype PKCS12 keytool -certreq -alias flink.rest -keystore rest.signed.keystore -storepass rest_keystore_password -keypass rest_key_password -file rest.csr @@ -252,6 +252,25 @@ security.ssl.rest.key-password: rest_key_password security.ssl.rest.truststore-password: ca_truststore_password {% endhighlight %} +**Tips to query REST Endpoint with curl utility** + +You can convert the keystore into the `PEM` format using `openssl`: + +{% highlight bash %} +openssl pkcs12 -passin pass:rest_keystore_password -in rest.keystore -out rest.pem -nodes +{% endhighlight %} + +Then you can query REST Endpoint with `curl`: + +{% highlight bash %} +curl --cacert rest.pem flink_url +{% endhighlight %} + +If mutual SSL is enabled: + +{% highlight bash %} +curl --cacert rest.pem --cert rest.pem flink_url +{% endhighlight %} ## Tips for YARN / Mesos Deployment diff --git a/docs/ops/state/large_state_tuning.md b/docs/ops/state/large_state_tuning.md index 62b3ee557f6d3..61956e5471453 100644 --- a/docs/ops/state/large_state_tuning.md +++ b/docs/ops/state/large_state_tuning.md @@ -161,21 +161,17 @@ RocksDBStateBackend.setOptions(new MyOptions()); public class MyOptions implements OptionsFactory { @Override - public DBOptions createDBOptions() { - return new DBOptions() - .setIncreaseParallelism(4) - .setUseFsync(false) - .setDisableDataSync(true); + public DBOptions createDBOptions(DBOptions currentOptions) { + return currentOptions.setIncreaseParallelism(4) + .setUseFsync(false); } - + @Override - public ColumnFamilyOptions createColumnOptions() { - - return new ColumnFamilyOptions() - .setTableFormatConfig( - new BlockBasedTableConfig() - .setBlockCacheSize(256 * 1024 * 1024) // 256 MB - .setBlockSize(128 * 1024)); // 128 KB + public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) { + return currentOptions.setTableFormatConfig( + new BlockBasedTableConfig() + .setBlockCacheSize(256 * 1024 * 1024) // 256 MB + .setBlockSize(128 * 1024)); // 128 KB } } {% endhighlight %} @@ -183,7 +179,7 @@ public class MyOptions implements OptionsFactory { **Predefined Options** Flink provides some predefined collections of option for RocksDB for different settings, which can be set for example via -`RocksDBStateBacked.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM)`. +`RocksDBStateBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM)`. We expect to accumulate more such profiles over time. Feel free to contribute such predefined option profiles when you found a set of options that work well and seem representative for certain workloads. @@ -324,7 +320,9 @@ and occupy local disk space. In the future, we might also offer an implementatio and occupy local disk space. For *incremental snapshots*, the local state is based on RocksDB's native checkpointing mechanism. This mechanism is also used as the first step to create the primary copy, which means that in this case no additional cost is introduced for creating the secondary copy. We simply keep the native checkpoint directory around instead of deleting it after uploading to the distributed store. This local copy can share active files with the working directory of RocksDB (via hard links), so for active -files also no additional disk space is consumed for task-local recovery with incremental snapshots. +files also no additional disk space is consumed for task-local recovery with incremental snapshots. Using hard links also means that the RocksDB directories must be on +the same physical device as all the configure local recovery directories that can be used to store local state, or else establishing hard links can fail (see FLINK-10954). +Currently, this also prevents using local recovery when RocksDB directories are configured to be located on more than one physical device. ### Allocation-preserving scheduling @@ -334,6 +332,4 @@ if a task manager is no longer available, a task that cannot return to its previ that the previous slot can only disappear when a task manager is no longer available, and in this case *some* tasks have to request a new slot anyways. With our scheduling strategy we give the maximum number of tasks a chance to recover from their local state and avoid the cascading effect of tasks stealing their previous slots from one another. -Allocation-preserving scheduling does not work with Flink's legacy mode. - {% top %} diff --git a/docs/ops/state/state_backends.md b/docs/ops/state/state_backends.md index b95d2b7e4dbe6..e4fe5bda06304 100644 --- a/docs/ops/state/state_backends.md +++ b/docs/ops/state/state_backends.md @@ -171,4 +171,8 @@ state.backend: filesystem state.checkpoints.dir: hdfs://namenode:40010/flink/checkpoints {% endhighlight %} +#### RocksDB State Backend Config Options + +{% include generated/rocks_db_configuration.html %} + {% top %} diff --git a/docs/ops/upgrading.md b/docs/ops/upgrading.md index eb00a2b28c6ba..22cea4c53f0d0 100644 --- a/docs/ops/upgrading.md +++ b/docs/ops/upgrading.md @@ -212,6 +212,7 @@ Savepoints are compatible across Flink versions as indicated by the table below: 1.4.x 1.5.x 1.6.x + 1.7.x Limitations @@ -224,6 +225,7 @@ Savepoints are compatible across Flink versions as indicated by the table below: + The maximum parallelism of a job that was migrated from Flink 1.1.x to 1.2.x+ is currently fixed as the parallelism of the job. This means that the parallelism can not be increased after migration. This limitation might be removed in a future bugfix release. @@ -236,6 +238,7 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O O + O When migrating from Flink 1.2.x to Flink 1.3.x+, changing parallelism at the same time is not supported. Users have to first take a savepoint after migrating to Flink 1.3.x+, and then change parallelism. Savepoints created for CEP applications cannot be restored in 1.4.x+. @@ -248,6 +251,7 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O O + O Migrating from Flink 1.3.0 to Flink 1.4.[0,1] will fail if the savepoint contains Scala case classes. Users have to directly migrate to 1.4.2+ instead. @@ -258,6 +262,7 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O O + O @@ -268,7 +273,11 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O - + O + There is a known issue with resuming broadcast state created with 1.5.x in versions + 1.6.x up to 1.6.2, and 1.7.0: FLINK-11087. Users + upgrading to 1.6.x or 1.7.x series need to directly migrate to minor versions higher than 1.6.2 and 1.7.0, + respectively. 1.6.x @@ -278,6 +287,18 @@ Savepoints are compatible across Flink versions as indicated by the table below: O + O + + + + 1.7.x + + + + + + + O diff --git a/docs/release-notes/flink-1.5.md b/docs/release-notes/flink-1.5.md index 4cee5774cdffc..ed5f2c2ad2654 100644 --- a/docs/release-notes/flink-1.5.md +++ b/docs/release-notes/flink-1.5.md @@ -80,5 +80,13 @@ The Kinesis dependencies of Flink’s Kinesis connector have been updated to the 0.12.9 ``` + +### Limitations of failover strategies +Flink's non-default failover strategies are still a very experimental feature which come with a set of limitations. +You should only use this feature if you are executing a stateless streaming job. +In any other cases, it is highly recommended to remove the config option `jobmanager.execution.failover-strategy` from your `flink-conf.yaml` or set it to `"full"`. + +In order to avoid future problems, this feature has been removed from the documentation until it will be fixed. +See [FLINK-10880](https://issues.apache.org/jira/browse/FLINK-10880) for more details. {% top %} diff --git a/docs/release-notes/flink-1.6.md b/docs/release-notes/flink-1.6.md index 34cd613551106..e2f9cc110d21c 100644 --- a/docs/release-notes/flink-1.6.md +++ b/docs/release-notes/flink-1.6.md @@ -29,6 +29,15 @@ The default value of the slot idle timeout `slot.idle.timeout` is set to the def ### Changed ElasticSearch 5.x Sink API Previous APIs in the Flink ElasticSearch 5.x Sink's `RequestIndexer` interface have been deprecated in favor of new signatures. -When adding requests to the `RequestIndexer`, the requests now must be of type `IndexRequest`, `DeleteRequest`, or `UpdateRequest`, instead of the base `ActionRequest`. +When adding requests to the `RequestIndexer`, the requests now must be of type `IndexRequest`, `DeleteRequest`, or `UpdateRequest`, instead of the base `ActionRequest`. + + +### Limitations of failover strategies +Flink's non-default failover strategies are still a very experimental feature which come with a set of limitations. +You should only use this feature if you are executing a stateless streaming job. +In any other cases, it is highly recommended to remove the config option `jobmanager.execution.failover-strategy` from your `flink-conf.yaml` or set it to `"full"`. + +In order to avoid future problems, this feature has been removed from the documentation until it will be fixed. +See [FLINK-10880](https://issues.apache.org/jira/browse/FLINK-10880) for more details. {% top %} diff --git a/docs/release-notes/flink-1.7.md b/docs/release-notes/flink-1.7.md index ea6840e65c140..507436b3b62a3 100644 --- a/docs/release-notes/flink-1.7.md +++ b/docs/release-notes/flink-1.7.md @@ -20,6 +20,134 @@ specific language governing permissions and limitations under the License. --> -These release notes discuss important aspects, such as configuration, behavior, or dependencies, that changed between Flink 1.6 and Flink 1.7. Please read these notes carefully if you are planning to upgrade your Flink version to 1.7. +These release notes discuss important aspects, such as configuration, behavior, or dependencies, that changed between Flink 1.6 and Flink 1.7. Please read these notes carefully if you are planning to upgrade your Flink version to 1.7. + +### Scala 2.12 support + +When using Scala `2.12` you might have to add explicit type annotations in places where they were not required when using Scala `2.11`. +This is an excerpt from the `TransitiveClosureNaive.scala` example in the Flink code base that shows the changes that could be required. + +Previous code: +``` +val terminate = prevPaths + .coGroup(nextPaths) + .where(0).equalTo(0) { + (prev, next, out: Collector[(Long, Long)]) => { + val prevPaths = prev.toSet + for (n <- next) + if (!prevPaths.contains(n)) out.collect(n) + } +} +``` + +With Scala `2.12` you have to change it to: +``` +val terminate = prevPaths + .coGroup(nextPaths) + .where(0).equalTo(0) { + (prev: Iterator[(Long, Long)], next: Iterator[(Long, Long)], out: Collector[(Long, Long)]) => { + val prevPaths = prev.toSet + for (n <- next) + if (!prevPaths.contains(n)) out.collect(n) + } +} +``` + +The reason for this is that Scala `2.12` changes how lambdas are implemented. +They now use the lambda support using SAM interfaces introduced in Java 8. +This makes some method calls ambiguous because now both Scala-style lambdas and SAMs are candidates for methods were it was previously clear which method would be invoked. + +### State evolution + +Before Flink 1.7, serializer snapshots were implemented as a `TypeSerializerConfigSnapshot` (which is now deprecated, and will eventually be removed in the future to be fully replaced by the new `TypeSerializerSnapshot` interface introduced in 1.7). +Moreover, the responsibility of serializer schema compatibility checks lived within the `TypeSerializer`, implemented in the `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` method. + +To be future-proof and to have flexibility to migrate your state serializers and schema, it is highly recommended to migrate from the old abstractions. +Details and migration guides can be found [here](https://ci.apache.org/projects/flink/flink-docs-master/dev/stream/state/custom_serialization.html). + +### Removal of the legacy mode + +Flink no longer supports the legacy mode. +If you depend on this, then please use Flink `1.6.x`. + +### Savepoints being used for recovery + +Savepoints are now used while recovering. +Previously when using exactly-once sink one could get into problems with duplicate output data when a failure occurred after a savepoint was taken but before the next checkpoint occurred. +This results in the fact that savepoints are no longer exclusively under the control of the user. +Savepoint should not be moved nor deleted if there was no newer checkpoint or savepoint taken. + +### MetricQueryService runs in separate thread pool + +The metric query service runs now in its own `ActorSystem`. +It needs consequently to open a new port for the query services to communicate with each other. +The [query service port]({{site.baseurl}}/ops/config.html#metrics-internal-query-service-port) can be configured in `flink-conf.yaml`. + +### Granularity of latency metrics + +The default granularity for latency metrics has been modified. +To restore the previous behavior users have to explicitly set the [granularity]({{site.baseurl}}/ops/config.html#metrics-latency-granularity) to `subtask`. + +### Latency marker activation + +Latency metrics are now disabled by default, which will affect all jobs that do not explicitly set the `latencyTrackingInterval` via `ExecutionConfig#setLatencyTrackingInterval`. +To restore the previous default behavior users have to configure the [latency interval]({{site.baseurl}}/ops/config.html#metrics-latency-interval) in `flink-conf.yaml`. + +### Relocation of Hadoop's Netty dependency + +We now also relocate Hadoop's Netty dependency from `io.netty` to `org.apache.flink.hadoop.shaded.io.netty`. +You can now bundle your own version of Netty into your job but may no longer assume that `io.netty` is present in the `flink-shaded-hadoop2-uber-*.jar` file. + +### Local recovery fixed + +With the improvements to Flink's scheduling, it can no longer happen that recoveries require more slots than before if local recovery is enabled. +Consequently, we encourage our users to enable [local recovery]({{site.baseurl}}/ops/config.html#state-backend-local-recovery) in `flink-conf.yaml`. + +### Support for multi slot TaskManagers + +Flink now properly supports `TaskManagers` with multiple slots. +Consequently, `TaskManagers` can now be started with an arbitrary number of slots and it is no longer recommended to start them with a single slot. + +### StandaloneJobClusterEntrypoint generates JobGraph with fixed JobID + +The `StandaloneJobClusterEntrypoint`, which is launched by the script `standalone-job.sh` and used for the job-mode container images, now starts all jobs with a fixed `JobID`. +Thus, in order to run a cluster in HA mode, one needs to set a different [cluster id]({{site.baseurl}}/ops/config.html#high-availability-cluster-id) for each job/cluster. + + +### Scala shell does not work with Scala 2.12 + +Flink's Scala shell does not work with Scala 2.12. +Therefore, the module `flink-scala-shell` is not being released for Scala 2.12. + +See [FLINK-10911](https://issues.apache.org/jira/browse/FLINK-10911) for more details. + + +### Limitations of failover strategies +Flink's non-default failover strategies are still a very experimental feature which come with a set of limitations. +You should only use this feature if you are executing a stateless streaming job. +In any other cases, it is highly recommended to remove the config option `jobmanager.execution.failover-strategy` from your `flink-conf.yaml` or set it to `"full"`. + +In order to avoid future problems, this feature has been removed from the documentation until it will be fixed. +See [FLINK-10880](https://issues.apache.org/jira/browse/FLINK-10880) for more details. + +### SQL over window preceding clause + +The over window `preceding` clause is now optional. +It defaults to `UNBOUNDED` if not specified. + +### OperatorSnapshotUtil writes v2 snapshots + +Snapshots created with `OperatorSnapshotUtil` are now written in the savepoint format `v2`. + +### SBT projects and the MiniClusterResource + +If you have a `sbt` project which uses the `MiniClusterResource`, you now have to add the `flink-runtime` test-jar dependency explicitly via: + +`libraryDependencies += "org.apache.flink" %% "flink-runtime" % flinkVersion % Test classifier "tests"` + +The reason for this is that the `MiniClusterResource` has been moved from `flink-test-utils` to `flink-runtime`. +The module `flink-test-utils` has correctly a `test-jar` dependency on `flink-runtime`. +However, `sbt` does not properly pull in transitive `test-jar` dependencies as described in this [sbt issue](https://github.com/sbt/sbt/issues/2964). +Consequently, it is necessary to specify the `test-jar` dependency explicitly. {% top %} diff --git a/docs/tutorials/datastream_api.md b/docs/tutorials/datastream_api.md index 4f93646c6fe45..b0964ee564ac5 100644 --- a/docs/tutorials/datastream_api.md +++ b/docs/tutorials/datastream_api.md @@ -71,9 +71,7 @@ wiki-edits/ ├── java │   └── wikiedits │   ├── BatchJob.java - │   ├── SocketTextStreamWordCount.java - │   ├── StreamingJob.java - │   └── WordCount.java + │   └── StreamingJob.java └── resources └── log4j.properties {% endhighlight %} @@ -294,7 +292,7 @@ your own machine and writing results to [Kafka](http://kafka.apache.org). ## Bonus Exercise: Running on a Cluster and Writing to Kafka Please follow our [local setup tutorial](local_setup.html) for setting up a Flink distribution -on your machine and refer to the [Kafka quickstart](https://kafka.apache.org/documentation.html#quickstart) +on your machine and refer to the [Kafka quickstart](https://kafka.apache.org/0110/documentation.html#quickstart) for setting up a Kafka installation before we proceed. As a first step, we have to add the Flink Kafka connector as a dependency so that we can @@ -303,7 +301,7 @@ use the Kafka sink. Add this to the `pom.xml` file in the dependencies section: {% highlight xml %} org.apache.flink - flink-connector-kafka-0.8_2.11 + flink-connector-kafka-0.11_2.11 ${flink.version} {% endhighlight %} @@ -320,12 +318,12 @@ result return tuple.toString(); } }) - .addSink(new FlinkKafkaProducer08<>("localhost:9092", "wiki-result", new SimpleStringSchema())); + .addSink(new FlinkKafkaProducer011<>("localhost:9092", "wiki-result", new SimpleStringSchema())); {% endhighlight %} The related classes also need to be imported: {% highlight java %} -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer08; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.api.common.serialization.SimpleStringSchema; import org.apache.flink.api.common.functions.MapFunction; {% endhighlight %} @@ -355,7 +353,7 @@ We also have to create the Kafka Topic, so that our program can write to it: {% highlight bash %} $ cd my/kafka/directory -$ bin/kafka-topics.sh --create --zookeeper localhost:2181 --topic wiki-results +$ bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic wiki-results {% endhighlight %} Now we are ready to run our jar file on the local Flink cluster: diff --git a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java index c193c4e055499..aed0d2975f490 100644 --- a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java +++ b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java @@ -59,6 +59,19 @@ public final class Documentation { int position() default Integer.MAX_VALUE; } + /** + * Annotation used on config option fields to exclude the config option from documentation. + */ + @Target(ElementType.FIELD) + @Retention(RetentionPolicy.RUNTIME) + @Internal + public @interface ExcludeFromDocumentation { + /** + * The optional reason why the config option is excluded from documentation. + */ + String value() default ""; + } + private Documentation(){ } } diff --git a/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..a0449ff300703 --- /dev/null +++ b/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE @@ -0,0 +1,16 @@ +flink-connector-cassandra +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.datastax.cassandra:cassandra-driver-core:3.0.0 +- com.datastax.cassandra:cassandra-driver-mapping:3.0.0 +- com.google.guava:guava:18.0 +- io.netty:netty-handler:4.0.33.Final +- io.netty:netty-buffer:4.0.33.Final +- io.netty:netty-common:4.0.33.Final +- io.netty:netty-transport:4.0.33.Final +- io.netty:netty-codec:4.0.33.Final diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/BufferingNoOpRequestIndexer.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/BufferingNoOpRequestIndexer.java new file mode 100644 index 0000000000000..e639b820ae317 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/BufferingNoOpRequestIndexer.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch; + +import org.apache.flink.annotation.Internal; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; + +import javax.annotation.concurrent.NotThreadSafe; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Implementation of a {@link RequestIndexer} that buffers {@link ActionRequest ActionRequests} + * before re-sending them to the Elasticsearch cluster upon request. + */ +@Internal +@NotThreadSafe +class BufferingNoOpRequestIndexer implements RequestIndexer { + + private List bufferedRequests; + + BufferingNoOpRequestIndexer() { + this.bufferedRequests = new ArrayList<>(10); + } + + @Override + public void add(DeleteRequest... deleteRequests) { + Collections.addAll(bufferedRequests, deleteRequests); + } + + @Override + public void add(IndexRequest... indexRequests) { + Collections.addAll(bufferedRequests, indexRequests); + } + + @Override + public void add(UpdateRequest... updateRequests) { + Collections.addAll(bufferedRequests, updateRequests); + } + + void processBufferedRequests(RequestIndexer actualIndexer) { + for (ActionRequest request : bufferedRequests) { + if (request instanceof IndexRequest) { + actualIndexer.add((IndexRequest) request); + } else if (request instanceof DeleteRequest) { + actualIndexer.add((DeleteRequest) request); + } else if (request instanceof UpdateRequest) { + actualIndexer.add((UpdateRequest) request); + } + } + + bufferedRequests.clear(); + } +} diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchSinkBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchSinkBase.java index 4d0c00252d279..da06d1c3a0878 100644 --- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchSinkBase.java +++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchSinkBase.java @@ -166,6 +166,9 @@ public void setDelayMillis(long delayMillis) { /** Provided to the user via the {@link ElasticsearchSinkFunction} to add {@link ActionRequest ActionRequests}. */ private transient RequestIndexer requestIndexer; + /** Provided to the {@link ActionRequestFailureHandler} to allow users to re-index failed requests. */ + private transient BufferingNoOpRequestIndexer failureRequestIndexer; + // ------------------------------------------------------------------------ // Internals for the Flink Elasticsearch Sink // ------------------------------------------------------------------------ @@ -200,7 +203,7 @@ public void setDelayMillis(long delayMillis) { private final AtomicReference failureThrowable = new AtomicReference<>(); public ElasticsearchSinkBase( - ElasticsearchApiCallBridge callBridge, + ElasticsearchApiCallBridge callBridge, Map userConfig, ElasticsearchSinkFunction elasticsearchSinkFunction, ActionRequestFailureHandler failureHandler) { @@ -296,13 +299,12 @@ public void open(Configuration parameters) throws Exception { client = callBridge.createClient(userConfig); bulkProcessor = buildBulkProcessor(new BulkProcessorListener()); requestIndexer = callBridge.createBulkProcessorIndexer(bulkProcessor, flushOnCheckpoint, numPendingRequests); + failureRequestIndexer = new BufferingNoOpRequestIndexer(); } @Override - public void invoke(T value) throws Exception { - // if bulk processor callbacks have previously reported an error, we rethrow the error and fail the sink - checkErrorAndRethrow(); - + public void invoke(T value, Context context) throws Exception { + checkAsyncErrorsAndRequests(); elasticsearchSinkFunction.process(value, getRuntimeContext(), requestIndexer); } @@ -313,13 +315,13 @@ public void initializeState(FunctionInitializationContext context) throws Except @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { - checkErrorAndRethrow(); + checkAsyncErrorsAndRequests(); if (flushOnCheckpoint) { - do { + while (numPendingRequests.get() != 0) { bulkProcessor.flush(); - checkErrorAndRethrow(); - } while (numPendingRequests.get() != 0); + checkAsyncErrorsAndRequests(); + } } } @@ -380,6 +382,11 @@ private void checkErrorAndRethrow() { } } + private void checkAsyncErrorsAndRequests() { + checkErrorAndRethrow(); + failureRequestIndexer.processBufferedRequests(requestIndexer); + } + private class BulkProcessorListener implements BulkProcessor.Listener { @Override public void beforeBulk(long executionId, BulkRequest request) { } @@ -400,9 +407,9 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon restStatus = itemResponse.getFailure().getStatus(); if (restStatus == null) { - failureHandler.onFailure(request.requests().get(i), failure, -1, requestIndexer); + failureHandler.onFailure(request.requests().get(i), failure, -1, failureRequestIndexer); } else { - failureHandler.onFailure(request.requests().get(i), failure, restStatus.getStatus(), requestIndexer); + failureHandler.onFailure(request.requests().get(i), failure, restStatus.getStatus(), failureRequestIndexer); } } } @@ -424,7 +431,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) try { for (ActionRequest action : request.requests()) { - failureHandler.onFailure(action, failure, -1, requestIndexer); + failureHandler.onFailure(action, failure, -1, failureRequestIndexer); } } catch (Throwable t) { // fail the sink and skip the rest of the items diff --git a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE index f588e0e873c14..a58d21ae6c6b0 100644 --- a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE @@ -1,109 +1,28 @@ -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 4.10.4 - - org.apache.lucene : lucene-analyzers-common version 4.10.4 - - org.apache.lucene : lucene-grouping version 4.10.4 - - org.apache.lucene : lucene-highlighter version 4.10.4 - - org.apache.lucene : lucene-join version 4.10.4 - - org.apache.lucene : lucene-memory version 4.10.4 - - org.apache.lucene : lucene-misc version 4.10.4 - - org.apache.lucene : lucene-queries version 4.10.4 - - org.apache.lucene : lucene-queryparser version 4.10.4 - - org.apache.lucene : lucene-sandbox version 4.10.4 - - org.apache.lucene : lucene-spatial version 4.10.4 - - org.apache.lucene : lucene-suggest version 4.10.4 - - com.spatial4j : spatial4j version 0.4.1 - - com.fasterxml.jackson.core : jackson-core version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.5.3 - - org.joda : joda-convert (copied classes) - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.12) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. +flink-connector-elasticsearch +Copyright 2014-2018 The Apache Software Foundation -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for Antlr -=================================== - -This project bundles antlr-runtime (v. 3.5) under the BSD 2-Clause License - -Copyright (c) 2010 Terence Parr - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2010 Terence Parr - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.spatial4j:spatial4j:0.4.1 +- org.apache.lucene:lucene-analyzers-common:4.10.4 +- org.apache.lucene:lucene-core:4.10.4 +- org.apache.lucene:lucene-grouping:4.10.4 +- org.apache.lucene:lucene-highlighter:4.10.4 +- org.apache.lucene:lucene-join:4.10.4 +- org.apache.lucene:lucene-memory:4.10.4 +- org.apache.lucene:lucene-misc:4.10.4 +- org.apache.lucene:lucene-queries:4.10.4 +- org.apache.lucene:lucene-queryparser:4.10.4 +- org.apache.lucene:lucene-sandbox:4.10.4 +- org.apache.lucene:lucene-spatial:4.10.4 +- org.apache.lucene:lucene-suggest:4.10.4 +- org.elasticsearch:elasticsearch:1.7.1 +- org.yaml:snakeyaml:1.12 + +This project bundles the following dependencies under the BSD License. +See bundled licenses for details. + +- org.antlr:antlr-runtime:jar:3.5 diff --git a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr new file mode 100644 index 0000000000000..6d149604e134b --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr @@ -0,0 +1,29 @@ +Copyright (c) 2010 Terence Parr + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2010 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml index 29fbf3ea33874..73d9e977758da 100644 --- a/flink-connectors/flink-connector-elasticsearch2/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml @@ -194,6 +194,24 @@ under the License. META-INF/maven/org.y*/** + + + *:* + + META-INF/LICENSE.txt + + + + io.netty:netty + + + META-INF/license + + META-INF/NOTICE.txt + + diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java b/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java index 73a69ebde3470..34f2ad3a49e5a 100644 --- a/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java @@ -21,6 +21,7 @@ import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; import org.apache.flink.streaming.connectors.elasticsearch.util.ElasticsearchUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.Preconditions; import org.elasticsearch.action.bulk.BackoffPolicy; @@ -72,6 +73,10 @@ public TransportClient createClient(Map clientConfig) { // verify that we actually are connected to a cluster if (transportClient.connectedNodes().isEmpty()) { + + // close the transportClient here + IOUtils.closeQuietly(transportClient); + throw new RuntimeException("Elasticsearch client is not connected to any Elasticsearch nodes!"); } diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE index c3d95ca99c016..4288bc1b48174 100644 --- a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE @@ -1,133 +1,83 @@ -This project includes software developed at +flink-connector-elasticsearch2 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 5.5.0 - - org.apache.lucene : lucene-analyzers-common version 5.5.0 - - org.apache.lucene : lucene-backward-codecs version 5.5.0 - - org.apache.lucene : lucene-grouping version 5.5.0 - - org.apache.lucene : lucene-highlighter version 5.5.0 - - org.apache.lucene : lucene-join version 5.5.0 - - org.apache.lucene : lucene-memory version 5.5.0 - - org.apache.lucene : lucene-misc version 5.5.0 - - org.apache.lucene : lucene-queries version 5.5.0 - - org.apache.lucene : lucene-queryparser version 5.5.0 - - org.apache.lucene : lucene-sandbox version 5.5.0 - - org.apache.lucene : lucene-spatial version 5.5.0 - - org.apache.lucene : lucene-spatial-extras version 5.5.0 - - org.apache.lucene : lucene-spatial3d version 5.5.0 - - org.apache.lucene : lucene-suggest version 5.5.0 - - com.carrotsearch : hppc version 0.7.1 - - com.google.guava : guava version 18.0 - - com.google.code.findbugs : jsr305 version 1.3.9 - - com.ning : compress-lzf version 1.0.2 - - com.spatial4j : spatial4j version 0.5 - - com.twitter : chill-java version 0.7.4 - - com.fasterxml.jackson.core : jackson-core version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.6.6 - - com.tdunning : t-digest version 3.0 - - io.netty : netty version 3.10.5.Final - - joda-time : joda-time version 2.5 - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.15) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. - -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for HdrHistogram -=================================== - -This project bundles HdrHistogram (v. 2.1.9) under the BSD 2-Clause License - -Original source repository: https://github.com/HdrHistogram/HdrHistogram - -The code in this repository code was Written by Gil Tene, Michael Barker, -and Matt Warren, and released to the public domain, as explained at -http://creativecommons.org/publicdomain/zero/1.0/ - -For users of this code who wish to consume it under the "BSD" license -rather than under the public domain or CC0 contribution text mentioned -above, the code found under this directory is *also* provided under the -following license (commonly referred to as the BSD 2-Clause License). This -license does not detract from the above stated release of the code into -the public domain, and simply represents an additional license granted by -the Author. - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene - Copyright (c) 2014 Michael Barker - Copyright (c) 2014 Matt Warren - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.carrotsearch:hppc:0.7.1 +- com.fasterxml.jackson.core:jackson-core:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.6.6 +- com.google.guava:guava:18.0 +- com.ning:compress-lzf:1.0.2 +- com.spatial4j:spatial4j:0.5 +- com.tdunning:t-digest:3.0 +- commons-cli:commons-cli:1.3.1 +- io.netty:netty:3.10.5.Final +- joda-time:joda-time:2.5 +- org.apache.lucene:lucene-analyzers-common:5.5.0 +- org.apache.lucene:lucene-backward-codecs:5.5.0 +- org.apache.lucene:lucene-core:5.5.0 +- org.apache.lucene:lucene-grouping:5.5.0 +- org.apache.lucene:lucene-highlighter:5.5.0 +- org.apache.lucene:lucene-join:5.5.0 +- org.apache.lucene:lucene-memory:5.5.0 +- org.apache.lucene:lucene-misc:5.5.0 +- org.apache.lucene:lucene-queries:5.5.0 +- org.apache.lucene:lucene-queryparser:5.5.0 +- org.apache.lucene:lucene-sandbox:5.5.0 +- org.apache.lucene:lucene-spatial:5.5.0 +- org.apache.lucene:lucene-spatial3d:5.5.0 +- org.apache.lucene:lucene-suggest:5.5.0 +- org.elasticsearch:elasticsearch:2.3.5 +- org.elasticsearch:securesm:1.0 +- org.joda:joda-convert:1.7 +- org.yaml:snakeyaml:1.15 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.6 + +This project bundles the following dependency under the CC0 1.0. +See bundled licenses for details. + +- com.twitter:jsr166e:1.1.0 + +This project bundles io.netty:netty:3.10.5.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 0000000000000..31ebc840539c1 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram new file mode 100644 index 0000000000000..09c38ea51608e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e new file mode 100644 index 0000000000000..0e259d42c9967 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 0000000000000..b1c292b54cb2b --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 0000000000000..29ad562af0a5e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 0000000000000..ec5f348998bd4 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml index a8a6489f22c4a..4ec8f8b770945 100644 --- a/flink-connectors/flink-connector-elasticsearch5/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml @@ -231,6 +231,13 @@ under the License. + + + *:* + + META-INF/LICENSE.txt + + * @@ -263,6 +270,17 @@ under the License. META-INF/log4j-provider.properties + + io.netty:netty + + + META-INF/license + + META-INF/NOTICE.txt + + diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java b/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java index a3453ec44452a..d59b8e9f912ff 100644 --- a/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java @@ -21,6 +21,7 @@ import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; import org.apache.flink.streaming.connectors.elasticsearch.util.ElasticsearchUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.Preconditions; import org.elasticsearch.action.bulk.BackoffPolicy; @@ -78,6 +79,10 @@ public TransportClient createClient(Map clientConfig) { // verify that we actually are connected to a cluster if (transportClient.connectedNodes().isEmpty()) { + + // close the transportClient here + IOUtils.closeQuietly(transportClient); + throw new RuntimeException("Elasticsearch client is not connected to any Elasticsearch nodes!"); } diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE index 19e96faa46003..9fe894149aed0 100644 --- a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE @@ -1,201 +1,104 @@ -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). +flink-connector-elasticsearch5 +Copyright 2014-2018 The Apache Software Foundation -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 6.3.0 - - org.apache.lucene : lucene-analyzers-common version 6.3.0 - - org.apache.lucene : lucene-backward-codecs version 6.3.0 - - org.apache.lucene : lucene-grouping version 6.3.0 - - org.apache.lucene : lucene-highlighter version 6.3.0 - - org.apache.lucene : lucene-join version 6.3.0 - - org.apache.lucene : lucene-memory version 6.3.0 - - org.apache.lucene : lucene-misc version 6.3.0 - - org.apache.lucene : lucene-queries version 6.3.0 - - org.apache.lucene : lucene-queryparser version 6.3.0 - - org.apache.lucene : lucene-sandbox version 6.3.0 - - org.apache.lucene : lucene-spatial version 6.3.0 - - org.apache.lucene : lucene-spatial-extras version 6.3.0 - - org.apache.lucene : lucene-spatial3d version 6.3.0 - - org.apache.lucene : lucene-suggest version 6.3.0 - - org.apache.httpcomponents : httpclient version 4.5.3 - - org.apache.httpcomponents : httpcore version 4.4.6 - - org.apache.httpcomponents : httpasynclcient version 4.1.2 - - org.apache.httpcomponents : httpcore-nio version 4.4.5 - - com.carrotsearch : hppc version 0.7.1 - - com.fasterxml.jackson.core : jackson-core version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.8.1 - - com.tdunning : t-digest version 3.0 - - io.netty : netty version 3.10.6.Final - - io.netty : netty-buffer version 4.1.6.Final - - io.netty : netty-codec version 4.1.6.Final - - io.netty : netty-codec-http version 4.1.6.Final - - io.netty : netty-common version 4.1.6.Final - - io.netty : netty-handler version 4.1.6.Final - - io.netty : netty-resolver version 4.1.6.Final - - io.netty : netty-transport version 4.1.6.Final - - org.jboss.netty : netty version 3.2.0.Final - - joda-time : joda-time version 2.5 - - com.github.spullara.mustache.java : compiler version 0.9.3 - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.15) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. - -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for joptsimple -=================================== - -This project bundles joptsimple under the MIT License. - -Original source repository: https://github.com/jopt-simple/jopt-simple - -Copyright (c) 2004-2016 Paul R. Holser, Jr. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=================================== - Notice for scopt -=================================== - -This project bundles scopt (v. 3.5.0) underr the MIT License. - -Original source repository: https://github.com/scopt/scopt - -scopt - Copyright (c) scopt contributors - -See source files for details. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -``Software''), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.carrotsearch:hppc:0.7.1 +- com.fasterxml.jackson.core:jackson-core:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.1 +- com.github.spullara.mustache.java:compiler:0.9.3 +- com.tdunning:t-digest:3.0 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- io.netty:netty:3.10.6.Final +- io.netty:netty-buffer:4.1.6.Final +- io.netty:netty-codec:4.1.6.Final +- io.netty:netty-codec-http:4.1.6.Final +- io.netty:netty-common:4.1.6.Final +- io.netty:netty-handler:4.1.6.Final +- io.netty:netty-resolver:4.1.6.Final +- io.netty:netty-transport:4.1.6.Final +- joda.time:joda-time:2.5 +- net.java.dev.jna:jna:4.2.2 +- org.apache.httpcomponents:httpasynclcient:4.1.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpcore-nio:4.4.5 +- org.apache.logging.log4j:log4j-api:2.7 +- org.apache.logging.log4j:log4j-to-slf4j:2.7 +- org.apache.lucene:lucene-analyzers-common:6.3.0 +- org.apache.lucene:lucene-backward-codecs:6.3.0 +- org.apache.lucene:lucene-core:6.3.0 +- org.apache.lucene:lucene-grouping:6.3.0 +- org.apache.lucene:lucene-highlighter:6.3.0 +- org.apache.lucene:lucene-join:6.3.0 +- org.apache.lucene:lucene-memory:6.3.0 +- org.apache.lucene:lucene-misc:6.3.0 +- org.apache.lucene:lucene-queries:6.3.0 +- org.apache.lucene:lucene-queryparser:6.3.0 +- org.apache.lucene:lucene-sandbox:6.3.0 +- org.apache.lucene:lucene-spatial:6.3.0 +- org.apache.lucene:lucene-spatial-extras:6.3.0 +- org.apache.lucene:lucene-spatial3d:6.3.0 +- org.apache.lucene:lucene-suggest:6.3.0 +- org.elasticsearch:elasticsearch:5.1.2 +- org.elasticsearch:securesm:1.1 +- org.elasticsearch.client:rest:5.1.2 +- org.elasticsearch.client:transport:5.1.2 +- org.elasticsearch.plugin:lang-mustache-client:5.1.2 +- org.elasticsearch.plugin:percolator-client:5.1.2 +- org.elasticsearch.plugin:reindex-client:5.1.2 +- org.elasticsearch.plugin:transport-netty3-client:5.1.2 +- org.elasticsearch.plugin:transport-netty4-client:5.1.2 +- org.yaml:snakeyaml:1.15 + +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) + +- net.sf.jopt-simple:jopt-simple:5.0.2 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.6 + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit -THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -=================================== - Notice for HdrHistogram -=================================== - -This project bundles HdrHistogram (v. 2.1.9) under the BSD 2-Clause License - -Original source repository: https://github.com/HdrHistogram/HdrHistogram - -The code in this repository code was Written by Gil Tene, Michael Barker, -and Matt Warren, and released to the public domain, as explained at -http://creativecommons.org/publicdomain/zero/1.0/ - -For users of this code who wish to consume it under the "BSD" license -rather than under the public domain or CC0 contribution text mentioned -above, the code found under this directory is *also* provided under the -following license (commonly referred to as the BSD 2-Clause License). This -license does not detract from the above stated release of the code into -the public domain, and simply represents an additional license granted by -the Author. - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene - Copyright (c) 2014 Michael Barker - Copyright (c) 2014 Matt Warren - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 0000000000000..31ebc840539c1 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram new file mode 100644 index 0000000000000..09c38ea51608e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple new file mode 100644 index 0000000000000..ef77fc95e107e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2004-2015 Paul R. Holser, Jr. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 0000000000000..b1c292b54cb2b --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 0000000000000..29ad562af0a5e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 0000000000000..ec5f348998bd4 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch6/pom.xml b/flink-connectors/flink-connector-elasticsearch6/pom.xml index d1eb3d8b3d1ee..9fca691d0acac 100644 --- a/flink-connectors/flink-connector-elasticsearch6/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch6/pom.xml @@ -197,29 +197,57 @@ under the License. true sql-jar + + + *:* + + + + com.carrotsearch:hppc + com.tdunning:t-digest + joda-time:joda-time + net.sf.jopt-simple:jopt-simple + org.elasticsearch:jna + org.hdrhistogram:HdrHistogram + org.yaml:snakeyaml + + - *:* - - + org.elasticsearch:elasticsearch - com/carrotsearch/** - com/sun/** - com/tdunning/** config/** + modules.txt + plugins.txt + org/joda/** + + + + org.elasticsearch.client:elasticsearch-rest-high-level-client + forbidden/** - joptsimple/** + + + + org.apache.httpcomponents:httpclient + + mozilla/** + + + + org.apache.lucene:lucene-analyzers-common + + org/tartarus/** + + + + *:* + + + META-INF/versions/** META-INF/services/com.fasterxml.** META-INF/services/org.apache.lucene.** META-INF/services/org.elasticsearch.** - META-INF/versions/** - modules.txt - mozilla/** - org/HdrHistogram/** - org/joda/** - org/tartarus/** - org/yaml/** - plugins.txt diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml index 135dc59b65543..52c283e017ba3 100644 --- a/flink-connectors/flink-connector-kafka-0.10/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml @@ -53,6 +53,10 @@ under the License. org.apache.kafka kafka_${scala.binary.version} + + org.apache.kafka + kafka-clients + @@ -187,7 +191,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml index bf04aebb390fb..2c91a05863ddc 100644 --- a/flink-connectors/flink-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml @@ -53,6 +53,10 @@ under the License. org.apache.kafka kafka_${scala.binary.version} + + org.apache.kafka + kafka-clients + @@ -196,7 +200,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java index 1cf4acf54064e..3e7cf2b1aca58 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java @@ -45,6 +45,7 @@ import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.NetUtils; import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; @@ -671,6 +672,9 @@ public void close() throws FlinkKafka011Exception { } // make sure we propagate pending errors checkErroneous(); + pendingTransactions().forEach(transaction -> + IOUtils.closeQuietly(transaction.getValue().producer) + ); } // ------------------- Logic for handling checkpoint flushing -------------------------- // @@ -689,7 +693,7 @@ protected KafkaTransactionState beginTransaction() throws FlinkKafka011Exception if (currentTransaction != null && currentTransaction.producer != null) { return new KafkaTransactionState(currentTransaction.producer); } - return new KafkaTransactionState(initProducer(true)); + return new KafkaTransactionState(initNonTransactionalProducer(true)); default: throw new UnsupportedOperationException("Not implemented semantic"); } @@ -712,73 +716,49 @@ protected void preCommit(KafkaTransactionState transaction) throws FlinkKafka011 @Override protected void commit(KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: + if (transaction.isTransactional()) { + try { transaction.producer.commitTransaction(); + } finally { recycleTransactionalProducer(transaction.producer); - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + } } } @Override protected void recoverAndCommit(KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - try (FlinkKafkaProducer producer = - initTransactionalProducer(transaction.transactionalId, false)) { - producer.resumeTransaction(transaction.producerId, transaction.epoch); - producer.commitTransaction(); - } - catch (InvalidTxnStateException | ProducerFencedException ex) { - // That means we have committed this transaction before. - LOG.warn("Encountered error {} while recovering transaction {}. " + + if (transaction.isTransactional()) { + try ( + FlinkKafkaProducer producer = + initTransactionalProducer(transaction.transactionalId, false)) { + producer.resumeTransaction(transaction.producerId, transaction.epoch); + producer.commitTransaction(); + } catch (InvalidTxnStateException | ProducerFencedException ex) { + // That means we have committed this transaction before. + LOG.warn("Encountered error {} while recovering transaction {}. " + "Presumably this transaction has been already committed before", - ex, - transaction); - } - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + ex, + transaction); + } } } @Override protected void abort(KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - transaction.producer.abortTransaction(); - recycleTransactionalProducer(transaction.producer); - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + if (transaction.isTransactional()) { + transaction.producer.abortTransaction(); + recycleTransactionalProducer(transaction.producer); } } @Override protected void recoverAndAbort(KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - try (FlinkKafkaProducer producer = - initTransactionalProducer(transaction.transactionalId, false)) { - producer.initTransactions(); - } - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + if (transaction.isTransactional()) { + try ( + FlinkKafkaProducer producer = + initTransactionalProducer(transaction.transactionalId, false)) { + producer.initTransactions(); + } } } @@ -905,9 +885,7 @@ private void cleanUpUserContext() { private void resetAvailableTransactionalIdsPool(Collection transactionalIds) { availableTransactionalIds.clear(); - for (String transactionalId : transactionalIds) { - availableTransactionalIds.add(transactionalId); - } + availableTransactionalIds.addAll(transactionalIds); } // ----------------------------------- Utilities -------------------------- @@ -957,6 +935,11 @@ private FlinkKafkaProducer initTransactionalProducer(String tran return initProducer(registerMetrics); } + private FlinkKafkaProducer initNonTransactionalProducer(boolean registerMetrics) { + producerConfig.remove("transactional.id"); + return initProducer(registerMetrics); + } + private FlinkKafkaProducer initProducer(boolean registerMetrics) { FlinkKafkaProducer producer = new FlinkKafkaProducer<>(this.producerConfig); @@ -1075,7 +1058,7 @@ static class KafkaTransactionState { } KafkaTransactionState( - String transactionalId, + @Nullable String transactionalId, long producerId, short epoch, FlinkKafkaProducer producer) { @@ -1085,6 +1068,10 @@ static class KafkaTransactionState { this.producer = producer; } + boolean isTransactional() { + return transactionalId != null; + } + @Override public String toString() { return String.format( diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaProducer.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaProducer.java index 8faff38749fbd..fa672f03ccd8b 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaProducer.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaProducer.java @@ -210,6 +210,7 @@ public void resumeTransaction(long producerId, short epoch) { } } + @Nullable public String getTransactionalId() { return transactionalId; } diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java index 57b7e77dc7f8f..ca75b1a637baf 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java @@ -48,11 +48,13 @@ import java.util.stream.IntStream; import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic; +import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic.AT_LEAST_ONCE; +import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic.EXACTLY_ONCE; import static org.apache.flink.util.ExceptionUtils.findThrowable; import static org.apache.flink.util.Preconditions.checkState; import static org.hamcrest.Matchers.lessThan; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; /** * IT cases for the {@link FlinkKafkaProducer011}. @@ -87,7 +89,7 @@ public void resourceCleanUpNone() throws Exception { @Test public void resourceCleanUpAtLeastOnce() throws Exception { - resourceCleanUp(Semantic.AT_LEAST_ONCE); + resourceCleanUp(AT_LEAST_ONCE); } /** @@ -153,7 +155,7 @@ public void testRestoreToCheckpointAfterExceedingProducersPool() throws Exceptio testHarness2.open(); } - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42)); deleteTestTopic(topic); } catch (Exception ex) { @@ -183,7 +185,7 @@ public void testFlinkKafkaProducer011FailBeforeNotify() throws Exception { try { testHarness.processElement(44, 4); testHarness.snapshot(2, 5); - assertFalse(true); + fail(); } catch (Exception ex) { // expected @@ -201,7 +203,7 @@ public void testFlinkKafkaProducer011FailBeforeNotify() throws Exception { testHarness.initializeState(snapshot); testHarness.close(); - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @@ -216,7 +218,7 @@ public void testFlinkKafkaProducer011FailTransactionCoordinatorBeforeNotify() th topic, integerKeyedSerializationSchema, properties, - Semantic.EXACTLY_ONCE); + EXACTLY_ONCE); OneInputStreamOperatorTestHarness testHarness1 = new OneInputStreamOperatorTestHarness<>( new StreamSink<>(kafkaProducer), @@ -250,7 +252,7 @@ public void testFlinkKafkaProducer011FailTransactionCoordinatorBeforeNotify() th testHarness2.open(); } - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @@ -296,7 +298,7 @@ public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception { // - aborted transactions with records 44 and 45 // - committed transaction with record 46 // - pending transaction with record 47 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46)); testHarness.close(); deleteTestTopic(topic); @@ -345,7 +347,7 @@ public void testFailAndRecoverSameCheckpointTwice() throws Exception { //now we should have: // - records 42 and 43 in committed transactions // - aborted transactions with records 44 and 45 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @@ -367,7 +369,7 @@ public void testScaleDownBeforeFirstCheckpoint() throws Exception { preScaleDownParallelism, preScaleDownParallelism, subtaskIndex, - Semantic.EXACTLY_ONCE); + EXACTLY_ONCE); preScaleDownOperator.setup(); preScaleDownOperator.open(); @@ -382,7 +384,7 @@ public void testScaleDownBeforeFirstCheckpoint() throws Exception { // there might not be any close) // After previous failure simulate restarting application with smaller parallelism - OneInputStreamOperatorTestHarness postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, Semantic.EXACTLY_ONCE); + OneInputStreamOperatorTestHarness postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, EXACTLY_ONCE); postScaleDownOperator1.setup(); postScaleDownOperator1.open(); @@ -397,7 +399,7 @@ public void testScaleDownBeforeFirstCheckpoint() throws Exception { // - records 42, 43, 44 and 45 in aborted transactions // - committed transaction with record 46 // - pending transaction with record 47 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46)); postScaleDownOperator1.close(); // ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids. @@ -466,8 +468,7 @@ public void testScaleUpAfterScalingDown() throws Exception { createProperties(), topic, 0, - IntStream.range(0, parallelism1 + parallelism2 + parallelism3).boxed().collect(Collectors.toList()), - 30_000L); + IntStream.range(0, parallelism1 + parallelism2 + parallelism3).boxed().collect(Collectors.toList())); deleteTestTopic(topic); } @@ -483,7 +484,7 @@ private List repartitionAndExecute( for (int subtaskIndex = 0; subtaskIndex < parallelism; subtaskIndex++) { OneInputStreamOperatorTestHarness testHarness = - createTestHarness(topic, maxParallelism, parallelism, subtaskIndex, Semantic.EXACTLY_ONCE); + createTestHarness(topic, maxParallelism, parallelism, subtaskIndex, EXACTLY_ONCE); testHarnesses.add(testHarness); testHarness.setup(); @@ -539,7 +540,7 @@ public void testRecoverCommittedTransaction() throws Exception { testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0 testHarness.close(); - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42)); deleteTestTopic(topic); } @@ -566,6 +567,50 @@ public void testRunOutOfProducersInThePool() throws Exception { deleteTestTopic(topic); } + @Test + public void testMigrateFromAtLeastOnceToExactlyOnce() throws Exception { + String topic = "testMigrateFromAtLeastOnceToExactlyOnce"; + testRecoverWithChangeSemantics(topic, AT_LEAST_ONCE, EXACTLY_ONCE); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 44, 45)); + deleteTestTopic(topic); + } + + @Test + public void testMigrateFromAtExactlyOnceToAtLeastOnce() throws Exception { + String topic = "testMigrateFromExactlyOnceToAtLeastOnce"; + testRecoverWithChangeSemantics(topic, EXACTLY_ONCE, AT_LEAST_ONCE); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 45, 46, 47)); + deleteTestTopic(topic); + } + + private void testRecoverWithChangeSemantics( + String topic, + Semantic fromSemantic, + Semantic toSemantic) throws Exception { + OperatorSubtaskState producerSnapshot; + try (OneInputStreamOperatorTestHarness testHarness = createTestHarness(topic, fromSemantic)) { + testHarness.setup(); + testHarness.open(); + testHarness.processElement(42, 0); + testHarness.snapshot(0, 1); + testHarness.processElement(43, 2); + testHarness.notifyOfCompletedCheckpoint(0); + producerSnapshot = testHarness.snapshot(1, 3); + testHarness.processElement(44, 4); + } + + try (OneInputStreamOperatorTestHarness testHarness = createTestHarness(topic, toSemantic)) { + testHarness.setup(); + testHarness.initializeState(producerSnapshot); + testHarness.open(); + testHarness.processElement(45, 7); + testHarness.snapshot(2, 8); + testHarness.processElement(46, 9); + testHarness.notifyOfCompletedCheckpoint(2); + testHarness.processElement(47, 9); + } + } + // shut down a Kafka broker private void failBroker(int brokerId) { KafkaServer toShutDown = null; @@ -604,7 +649,13 @@ private void closeIgnoringProducerFenced(AutoCloseable autoCloseable) throws Exc } private OneInputStreamOperatorTestHarness createTestHarness(String topic) throws Exception { - return createTestHarness(topic, 1, 1, 0, Semantic.EXACTLY_ONCE); + return createTestHarness(topic, Semantic.EXACTLY_ONCE); + } + + private OneInputStreamOperatorTestHarness createTestHarness( + String topic, + Semantic semantic) throws Exception { + return createTestHarness(topic, 1, 1, 0, semantic); } private OneInputStreamOperatorTestHarness createTestHarness( diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml index 52f0ef17b9ad9..50f41d0815a69 100644 --- a/flink-connectors/flink-connector-kafka-0.8/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml @@ -176,7 +176,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml index 9b9bbefa7b3f7..197ce2d5f0721 100644 --- a/flink-connectors/flink-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml @@ -60,6 +60,10 @@ under the License. org.apache.kafka kafka_${scala.binary.version} + + org.apache.kafka + kafka-clients + @@ -137,7 +141,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml index a0feec90b55a3..4921550c0996b 100644 --- a/flink-connectors/flink-connector-kafka-base/pom.xml +++ b/flink-connectors/flink-connector-kafka-base/pom.xml @@ -153,7 +153,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index cfb5b6d510d0f..90f182a24c937 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -50,6 +50,7 @@ import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionStateSentinel; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.SerializedValue; import org.apache.commons.collections.map.LinkedMap; @@ -469,9 +470,7 @@ public void open(Configuration configuration) throws Exception { this.partitionDiscoverer.open(); subscribedPartitionsToStartOffsets = new HashMap<>(); - - List allPartitions = partitionDiscoverer.discoverPartitions(); - + final List allPartitions = partitionDiscoverer.discoverPartitions(); if (restoredState != null) { for (KafkaTopicPartition partition : allPartitions) { if (!restoredState.containsKey(partition)) { @@ -485,7 +484,7 @@ public void open(Configuration configuration) throws Exception { // restored partitions that should not be subscribed by this subtask if (KafkaTopicPartitionAssigner.assign( restoredStateEntry.getKey(), getRuntimeContext().getNumberOfParallelSubtasks()) - == getRuntimeContext().getIndexOfThisSubtask()){ + == getRuntimeContext().getIndexOfThisSubtask()){ subscribedPartitionsToStartOffsets.put(restoredStateEntry.getKey(), restoredStateEntry.getValue()); } } else { @@ -533,16 +532,16 @@ public void open(Configuration configuration) throws Exception { } for (Map.Entry partitionToOffset - : fetchOffsetsWithTimestamp(allPartitions, startupOffsetsTimestamp).entrySet()) { + : fetchOffsetsWithTimestamp(allPartitions, startupOffsetsTimestamp).entrySet()) { subscribedPartitionsToStartOffsets.put( partitionToOffset.getKey(), (partitionToOffset.getValue() == null) - // if an offset cannot be retrieved for a partition with the given timestamp, - // we default to using the latest offset for the partition - ? KafkaTopicPartitionStateSentinel.LATEST_OFFSET - // since the specified offsets represent the next record to read, we subtract - // it by one so that the initial state of the consumer will be correct - : partitionToOffset.getValue() - 1); + // if an offset cannot be retrieved for a partition with the given timestamp, + // we default to using the latest offset for the partition + ? KafkaTopicPartitionStateSentinel.LATEST_OFFSET + // since the specified offsets represent the next record to read, we subtract + // it by one so that the initial state of the consumer will be correct + : partitionToOffset.getValue() - 1); } break; @@ -595,7 +594,6 @@ public void open(Configuration configuration) throws Exception { partitionsDefaultedToGroupOffsets); } break; - default: case GROUP_OFFSETS: LOG.info("Consumer subtask {} will start reading the following {} partitions from the committed group offsets in Kafka: {}", getRuntimeContext().getIndexOfThisSubtask(), @@ -663,80 +661,87 @@ public void onException(Throwable cause) { // 1) New state - partition discovery loop executed as separate thread, with this // thread running the main fetcher loop // 2) Old state - partition discovery is disabled and only the main fetcher loop is executed + if (discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED) { + kafkaFetcher.runFetchLoop(); + } else { + runWithPartitionDiscovery(); + } + } - if (discoveryIntervalMillis != PARTITION_DISCOVERY_DISABLED) { - final AtomicReference discoveryLoopErrorRef = new AtomicReference<>(); - this.discoveryLoopThread = new Thread(new Runnable() { - @Override - public void run() { - try { - // --------------------- partition discovery loop --------------------- + private void runWithPartitionDiscovery() throws Exception { + final AtomicReference discoveryLoopErrorRef = new AtomicReference<>(); + createAndStartDiscoveryLoop(discoveryLoopErrorRef); - List discoveredPartitions; + kafkaFetcher.runFetchLoop(); - // throughout the loop, we always eagerly check if we are still running before - // performing the next operation, so that we can escape the loop as soon as possible + // make sure that the partition discoverer is waked up so that + // the discoveryLoopThread exits + partitionDiscoverer.wakeup(); + joinDiscoveryLoopThread(); - while (running) { - if (LOG.isDebugEnabled()) { - LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask()); - } + // rethrow any fetcher errors + final Exception discoveryLoopError = discoveryLoopErrorRef.get(); + if (discoveryLoopError != null) { + throw new RuntimeException(discoveryLoopError); + } + } - try { - discoveredPartitions = partitionDiscoverer.discoverPartitions(); - } catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) { - // the partition discoverer may have been closed or woken up before or during the discovery; - // this would only happen if the consumer was canceled; simply escape the loop - break; - } + @VisibleForTesting + void joinDiscoveryLoopThread() throws InterruptedException { + if (discoveryLoopThread != null) { + discoveryLoopThread.join(); + } + } - // no need to add the discovered partitions if we were closed during the meantime - if (running && !discoveredPartitions.isEmpty()) { - kafkaFetcher.addDiscoveredPartitions(discoveredPartitions); - } + private void createAndStartDiscoveryLoop(AtomicReference discoveryLoopErrorRef) { + discoveryLoopThread = new Thread(() -> { + try { + // --------------------- partition discovery loop --------------------- - // do not waste any time sleeping if we're not running anymore - if (running && discoveryIntervalMillis != 0) { - try { - Thread.sleep(discoveryIntervalMillis); - } catch (InterruptedException iex) { - // may be interrupted if the consumer was canceled midway; simply escape the loop - break; - } - } - } - } catch (Exception e) { - discoveryLoopErrorRef.set(e); - } finally { - // calling cancel will also let the fetcher loop escape - // (if not running, cancel() was already called) - if (running) { - cancel(); - } - } - } - }, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks()); + // throughout the loop, we always eagerly check if we are still running before + // performing the next operation, so that we can escape the loop as soon as possible - discoveryLoopThread.start(); - kafkaFetcher.runFetchLoop(); + while (running) { + if (LOG.isDebugEnabled()) { + LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask()); + } - // -------------------------------------------------------------------- + final List discoveredPartitions; + try { + discoveredPartitions = partitionDiscoverer.discoverPartitions(); + } catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) { + // the partition discoverer may have been closed or woken up before or during the discovery; + // this would only happen if the consumer was canceled; simply escape the loop + break; + } - // make sure that the partition discoverer is properly closed - partitionDiscoverer.close(); - discoveryLoopThread.join(); + // no need to add the discovered partitions if we were closed during the meantime + if (running && !discoveredPartitions.isEmpty()) { + kafkaFetcher.addDiscoveredPartitions(discoveredPartitions); + } - // rethrow any fetcher errors - final Exception discoveryLoopError = discoveryLoopErrorRef.get(); - if (discoveryLoopError != null) { - throw new RuntimeException(discoveryLoopError); + // do not waste any time sleeping if we're not running anymore + if (running && discoveryIntervalMillis != 0) { + try { + Thread.sleep(discoveryIntervalMillis); + } catch (InterruptedException iex) { + // may be interrupted if the consumer was canceled midway; simply escape the loop + break; + } + } + } + } catch (Exception e) { + discoveryLoopErrorRef.set(e); + } finally { + // calling cancel will also let the fetcher loop escape + // (if not running, cancel() was already called) + if (running) { + cancel(); + } } - } else { - // won't be using the discoverer - partitionDiscoverer.close(); + }, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks()); - kafkaFetcher.runFetchLoop(); - } + discoveryLoopThread.start(); } @Override @@ -766,11 +771,27 @@ public void cancel() { @Override public void close() throws Exception { - // pretty much the same logic as cancelling + cancel(); + + joinDiscoveryLoopThread(); + + Exception exception = null; + if (partitionDiscoverer != null) { + try { + partitionDiscoverer.close(); + } catch (Exception e) { + exception = e; + } + } + try { - cancel(); - } finally { super.close(); + } catch (Exception e) { + exception = ExceptionUtils.firstOrSuppressed(e, exception); + } + + if (exception != null) { + throw exception; } } diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java index 86d7ef6d35ba3..0dbbbeb348a01 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java @@ -21,9 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.streaming.connectors.kafka.config.StartupMode; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -40,7 +38,7 @@ public class KafkaValidator extends ConnectorDescriptorValidator { public static final String CONNECTOR_VERSION_VALUE_09 = "0.9"; public static final String CONNECTOR_VERSION_VALUE_010 = "0.10"; public static final String CONNECTOR_VERSION_VALUE_011 = "0.11"; - public static final String CONNECTOR_VERSION_VALUE_20 = "2.0"; + public static final String CONNECTOR_VERSION_VALUE_UNIVERSAL = "universal"; public static final String CONNECTOR_TOPIC = "connector.topic"; public static final String CONNECTOR_STARTUP_MODE = "connector.startup-mode"; public static final String CONNECTOR_STARTUP_MODE_VALUE_EARLIEST = "earliest-offset"; @@ -64,7 +62,7 @@ public void validate(DescriptorProperties properties) { super.validate(properties); properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_KAFKA, false); - validateVersion(properties); + properties.validateString(CONNECTOR_TOPIC, false, 1, Integer.MAX_VALUE); validateStartupMode(properties); @@ -73,17 +71,6 @@ public void validate(DescriptorProperties properties) { validateSinkPartitioner(properties); } - private void validateVersion(DescriptorProperties properties) { - final List versions = Arrays.asList( - CONNECTOR_VERSION_VALUE_08, - CONNECTOR_VERSION_VALUE_09, - CONNECTOR_VERSION_VALUE_010, - CONNECTOR_VERSION_VALUE_011, - CONNECTOR_VERSION_VALUE_20); - properties.validateEnumValues(CONNECTOR_VERSION, false, versions); - properties.validateString(CONNECTOR_TOPIC, false, 1, Integer.MAX_VALUE); - } - private void validateStartupMode(DescriptorProperties properties) { final Map> specificOffsetValidators = new HashMap<>(); specificOffsetValidators.put( diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java index dbe76302ab0a8..1775c4c455384 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java @@ -47,12 +47,18 @@ import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor; import org.apache.flink.streaming.connectors.kafka.testutils.TestPartitionDiscoverer; import org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService; import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; import org.apache.flink.streaming.util.MockStreamingRuntimeContext; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.function.SupplierWithException; +import org.apache.flink.util.function.ThrowingRunnable; import org.junit.Assert; import org.junit.Test; @@ -70,10 +76,12 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import static org.apache.flink.util.Preconditions.checkState; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.collection.IsIn.isIn; import static org.hamcrest.collection.IsMapContaining.hasKey; import static org.hamcrest.core.IsNot.not; @@ -83,12 +91,13 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; /** * Tests for the {@link FlinkKafkaConsumerBase}. */ -public class FlinkKafkaConsumerBaseTest { +public class FlinkKafkaConsumerBaseTest extends TestLogger { /** * Tests that not both types of timestamp extractors / watermark generators can be used. @@ -208,13 +217,7 @@ public void testConfigureAutoCommitMode() throws Exception { @SuppressWarnings("unchecked") final DummyFlinkKafkaConsumer consumer = new DummyFlinkKafkaConsumer<>(true); - setupConsumer( - consumer, - false, - null, - false, // disable checkpointing; auto commit should be respected - 0, - 1); + setupConsumer(consumer); assertEquals(OffsetCommitMode.KAFKA_PERIODIC, consumer.getOffsetCommitMode()); } @@ -242,13 +245,7 @@ public void testConfigureDisableOffsetCommitWithoutCheckpointing() throws Except @SuppressWarnings("unchecked") final DummyFlinkKafkaConsumer consumer = new DummyFlinkKafkaConsumer<>(false); - setupConsumer( - consumer, - false, - null, - false, // disable checkpointing; auto commit should be respected - 0, - 1); + setupConsumer(consumer); assertEquals(OffsetCommitMode.DISABLED, consumer.getOffsetCommitMode()); } @@ -464,6 +461,90 @@ public void go() throws Exception { runThread.sync(); } + @Test + public void testClosePartitionDiscovererWhenOpenThrowException() throws Exception { + final RuntimeException failureCause = new RuntimeException(new FlinkException("Test partition discoverer exception")); + final FailingPartitionDiscoverer failingPartitionDiscoverer = new FailingPartitionDiscoverer(failureCause); + + final DummyFlinkKafkaConsumer consumer = new DummyFlinkKafkaConsumer<>(failingPartitionDiscoverer); + + testFailingConsumerLifecycle(consumer, failureCause); + assertTrue("partitionDiscoverer should be closed when consumer is closed", failingPartitionDiscoverer.isClosed()); + } + + @Test + public void testClosePartitionDiscovererWhenCreateKafkaFetcherFails() throws Exception { + final FlinkException failureCause = new FlinkException("Create Kafka fetcher failure."); + + final DummyPartitionDiscoverer testPartitionDiscoverer = new DummyPartitionDiscoverer(); + final DummyFlinkKafkaConsumer consumer = new DummyFlinkKafkaConsumer<>( + () -> { + throw failureCause; + }, + testPartitionDiscoverer, + 100L); + + testFailingConsumerLifecycle(consumer, failureCause); + assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed()); + } + + @Test + public void testClosePartitionDiscovererWhenKafkaFetcherFails() throws Exception { + final FlinkException failureCause = new FlinkException("Run Kafka fetcher failure."); + + // in this scenario, the partition discoverer will be concurrently accessed; + // use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call + // wakeup() before closing the discoverer + final WakeupBeforeCloseTestingPartitionDiscoverer testPartitionDiscoverer = new WakeupBeforeCloseTestingPartitionDiscoverer(); + + final AbstractFetcher mock = (AbstractFetcher) mock(AbstractFetcher.class); + doThrow(failureCause).when(mock).runFetchLoop(); + + final DummyFlinkKafkaConsumer consumer = new DummyFlinkKafkaConsumer<>(() -> mock, testPartitionDiscoverer, 100L); + + testFailingConsumerLifecycle(consumer, failureCause); + assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed()); + } + + private void testFailingConsumerLifecycle(FlinkKafkaConsumerBase testKafkaConsumer, @Nonnull Exception expectedException) throws Exception { + try { + setupConsumer(testKafkaConsumer); + testKafkaConsumer.run(new TestSourceContext<>()); + + fail("Exception should have been thrown from open / run method of FlinkKafkaConsumerBase."); + } catch (Exception e) { + assertThat(ExceptionUtils.findThrowable(e, throwable -> throwable.equals(expectedException)).isPresent(), is(true)); + } + testKafkaConsumer.close(); + } + + @Test + public void testClosePartitionDiscovererWithCancellation() throws Exception { + final DummyPartitionDiscoverer testPartitionDiscoverer = new DummyPartitionDiscoverer(); + + final TestingFlinkKafkaConsumer consumer = new TestingFlinkKafkaConsumer<>(testPartitionDiscoverer, 100L); + + testNormalConsumerLifecycle(consumer); + assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed()); + } + + private void testNormalConsumerLifecycle(FlinkKafkaConsumerBase testKafkaConsumer) throws Exception { + setupConsumer(testKafkaConsumer); + final CompletableFuture runFuture = CompletableFuture.runAsync(ThrowingRunnable.unchecked(() -> testKafkaConsumer.run(new TestSourceContext<>()))); + testKafkaConsumer.close(); + runFuture.get(); + } + + private void setupConsumer(FlinkKafkaConsumerBase consumer) throws Exception { + setupConsumer( + consumer, + false, + null, + false, + 0, + 1); + } + @Test public void testScaleUp() throws Exception { testRescaling(5, 2, 8, 30); @@ -607,6 +688,155 @@ private static AbstractStreamOperatorTestHarness createTestHarness( // ------------------------------------------------------------------------ + /** + * A dummy partition discoverer that always throws an exception from discoverPartitions() method. + */ + private static class FailingPartitionDiscoverer extends AbstractPartitionDiscoverer { + + private volatile boolean closed = false; + + private final RuntimeException failureCause; + + public FailingPartitionDiscoverer(RuntimeException failureCause) { + super( + new KafkaTopicsDescriptor(Arrays.asList("foo"), null), + 0, + 1); + this.failureCause = failureCause; + } + + @Override + protected void initializeConnections() throws Exception { + closed = false; + } + + @Override + protected void wakeupConnections() { + + } + + @Override + protected void closeConnections() throws Exception { + closed = true; + } + + @Override + protected List getAllTopics() throws WakeupException { + return null; + } + + @Override + protected List getAllPartitionsForTopics(List topics) throws WakeupException { + return null; + } + + @Override public List discoverPartitions() throws WakeupException, ClosedException { + throw failureCause; + } + + public boolean isClosed() { + return closed; + } + } + + private static class WakeupBeforeCloseTestingPartitionDiscoverer extends DummyPartitionDiscoverer { + @Override + protected void closeConnections() { + if (!isWakedUp()) { + fail("Partition discoverer should have been waked up first before closing."); + } + + super.closeConnections(); + } + } + + private static class DummyPartitionDiscoverer extends AbstractPartitionDiscoverer { + + private final List allTopics; + private final List allPartitions; + private volatile boolean closed = false; + private volatile boolean wakedUp = false; + + private DummyPartitionDiscoverer() { + super(new KafkaTopicsDescriptor(Collections.singletonList("foo"), null), 0, 1); + this.allTopics = Collections.singletonList("foo"); + this.allPartitions = Collections.singletonList(new KafkaTopicPartition("foo", 0)); + } + + @Override + protected void initializeConnections() { + //noop + } + + @Override + protected void wakeupConnections() { + wakedUp = true; + } + + @Override + protected void closeConnections() { + closed = true; + } + + @Override + protected List getAllTopics() throws WakeupException { + checkState(); + + return allTopics; + } + + @Override + protected List getAllPartitionsForTopics(List topics) throws WakeupException { + checkState(); + return allPartitions; + } + + private void checkState() throws WakeupException { + if (wakedUp || closed) { + throw new WakeupException(); + } + } + + boolean isClosed() { + return closed; + } + + public boolean isWakedUp() { + return wakedUp; + } + } + + private static class TestingFetcher extends AbstractFetcher { + + private volatile boolean isRunning = true; + + protected TestingFetcher(SourceFunction.SourceContext sourceContext, Map seedPartitionsWithInitialOffsets, SerializedValue> watermarksPeriodic, SerializedValue> watermarksPunctuated, ProcessingTimeService processingTimeProvider, long autoWatermarkInterval, ClassLoader userCodeClassLoader, MetricGroup consumerMetricGroup, boolean useMetrics) throws Exception { + super(sourceContext, seedPartitionsWithInitialOffsets, watermarksPeriodic, watermarksPunctuated, processingTimeProvider, autoWatermarkInterval, userCodeClassLoader, consumerMetricGroup, useMetrics); + } + + @Override + public void runFetchLoop() throws Exception { + while (isRunning) { + Thread.sleep(10L); + } + } + + @Override + public void cancel() { + isRunning = false; + } + + @Override + protected void doCommitInternalOffsetsToKafka(Map offsets, @Nonnull KafkaCommitCallback commitCallback) throws Exception { + + } + + @Override + protected KPH createKafkaPartitionHandle(KafkaTopicPartition partition) { + return null; + } + } + /** * An instantiable dummy {@link FlinkKafkaConsumerBase} that supports injecting * mocks for {@link FlinkKafkaConsumerBase#kafkaFetcher}, {@link FlinkKafkaConsumerBase#partitionDiscoverer}, @@ -615,7 +845,7 @@ private static AbstractStreamOperatorTestHarness createTestHarness( private static class DummyFlinkKafkaConsumer extends FlinkKafkaConsumerBase { private static final long serialVersionUID = 1L; - private AbstractFetcher testFetcher; + private SupplierWithException, Exception> testFetcherSupplier; private AbstractPartitionDiscoverer testPartitionDiscoverer; private boolean isAutoCommitEnabled; @@ -629,20 +859,56 @@ private static class DummyFlinkKafkaConsumer extends FlinkKafkaConsumerBase, Exception> abstractFetcherSupplier, AbstractPartitionDiscoverer abstractPartitionDiscoverer, long discoveryIntervalMillis) { + this(abstractFetcherSupplier, abstractPartitionDiscoverer, false, discoveryIntervalMillis); + } + @SuppressWarnings("unchecked") DummyFlinkKafkaConsumer( AbstractFetcher testFetcher, AbstractPartitionDiscoverer testPartitionDiscoverer, boolean isAutoCommitEnabled) { + this( + testFetcher, + testPartitionDiscoverer, + isAutoCommitEnabled, + PARTITION_DISCOVERY_DISABLED); + } + + @SuppressWarnings("unchecked") + DummyFlinkKafkaConsumer( + AbstractFetcher testFetcher, + AbstractPartitionDiscoverer testPartitionDiscoverer, + boolean isAutoCommitEnabled, + long discoveryIntervalMillis) { + this( + () -> testFetcher, + testPartitionDiscoverer, + isAutoCommitEnabled, + discoveryIntervalMillis); + } + + @SuppressWarnings("unchecked") + DummyFlinkKafkaConsumer( + SupplierWithException, Exception> testFetcherSupplier, + AbstractPartitionDiscoverer testPartitionDiscoverer, + boolean isAutoCommitEnabled, + long discoveryIntervalMillis) { super( Collections.singletonList("dummy-topic"), null, (KeyedDeserializationSchema < T >) mock(KeyedDeserializationSchema.class), - PARTITION_DISCOVERY_DISABLED, + discoveryIntervalMillis, false); - this.testFetcher = testFetcher; + this.testFetcherSupplier = testFetcherSupplier; this.testPartitionDiscoverer = testPartitionDiscoverer; this.isAutoCommitEnabled = isAutoCommitEnabled; } @@ -658,7 +924,7 @@ private static class DummyFlinkKafkaConsumer extends FlinkKafkaConsumerBase fetchOffsetsWithTimestamp( } } + private static class TestingFlinkKafkaConsumer extends FlinkKafkaConsumerBase { + + private static final long serialVersionUID = 935384661907656996L; + + private final AbstractPartitionDiscoverer partitionDiscoverer; + + TestingFlinkKafkaConsumer(final AbstractPartitionDiscoverer partitionDiscoverer, long discoveryIntervalMillis) { + super(Collections.singletonList("dummy-topic"), + null, + (KeyedDeserializationSchema < T >) mock(KeyedDeserializationSchema.class), + discoveryIntervalMillis, + false); + this.partitionDiscoverer = partitionDiscoverer; + } + + @Override + protected AbstractFetcher createFetcher(SourceContext sourceContext, Map thisSubtaskPartitionsWithStartOffsets, SerializedValue> watermarksPeriodic, SerializedValue> watermarksPunctuated, StreamingRuntimeContext runtimeContext, OffsetCommitMode offsetCommitMode, MetricGroup consumerMetricGroup, boolean useMetrics) throws Exception { + return new TestingFetcher(sourceContext, thisSubtaskPartitionsWithStartOffsets, watermarksPeriodic, watermarksPunctuated, runtimeContext.getProcessingTimeService(), 0L, getClass().getClassLoader(), consumerMetricGroup, useMetrics); + + } + + @Override + protected AbstractPartitionDiscoverer createPartitionDiscoverer(KafkaTopicsDescriptor topicsDescriptor, int indexOfThisSubtask, int numParallelSubtasks) { + return partitionDiscoverer; + } + + @Override + protected boolean getIsAutoCommitEnabled() { + return false; + } + + @Override + protected Map fetchOffsetsWithTimestamp(Collection partitions, long timestamp) { + throw new UnsupportedOperationException("fetchOffsetsWithTimestamp is not supported"); + } + } + private static final class TestingListState implements ListState { private final List list = new ArrayList<>(); diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java index 69eb94a2e8f72..cf68c3de77635 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java @@ -329,7 +329,7 @@ protected void testExactlyOnce(boolean regularSink, int sinksCount) throws Excep } TypeInformationSerializationSchema schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig()); - KeyedSerializationSchema keyedSerializationSchema = new KeyedSerializationSchemaWrapper(schema); + KeyedSerializationSchema keyedSerializationSchema = new KeyedSerializationSchemaWrapper<>(schema); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500); diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java index 1a20d7e82adf4..2d4ac43fa9a71 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java @@ -241,6 +241,14 @@ protected void assertAtLeastOnceForTopic( fail(String.format("Expected to contain all of: <%s>, but was: <%s>", expectedElements, actualElements)); } + protected void assertExactlyOnceForTopic( + Properties properties, + String topic, + int partition, + List expectedElements) { + assertExactlyOnceForTopic(properties, topic, partition, expectedElements, 30_000L); + } + /** * We manually handle the timeout instead of using JUnit's timeout to return failure instead of timeout error. * After timeout we assume that there are missing records and there is a bug, not that the test has run out of time. @@ -250,7 +258,7 @@ protected void assertExactlyOnceForTopic( String topic, int partition, List expectedElements, - long timeoutMillis) throws Exception { + long timeoutMillis) { long startMillis = System.currentTimeMillis(); List actualElements = new ArrayList<>(); diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml index 109535aa4db65..8619980fe8f0c 100644 --- a/flink-connectors/flink-connector-kafka/pom.xml +++ b/flink-connectors/flink-connector-kafka/pom.xml @@ -36,7 +36,7 @@ under the License. jar - 2.0.0 + 2.0.1 @@ -52,6 +52,10 @@ under the License. org.apache.kafka kafka_${scala.binary.version} + + org.apache.kafka + kafka-clients + @@ -136,28 +140,6 @@ under the License. org.apache.flink flink-connector-kafka-base_${scala.binary.version} ${project.version} - - - - org.apache.kafka - kafka_${scala.binary.version} - - - test-jar - test - - - - org.apache.flink - flink-connector-kafka-base_${scala.binary.version} - ${project.version} - - - - org.apache.kafka - kafka_${scala.binary.version} - - test-jar test @@ -201,7 +183,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} test diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java index 0ac2f906bde53..10e8ef1713dce 100644 --- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java @@ -45,6 +45,7 @@ import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.NetUtils; import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; @@ -673,6 +674,9 @@ public void close() throws FlinkKafkaException { } // make sure we propagate pending errors checkErroneous(); + pendingTransactions().forEach(transaction -> + IOUtils.closeQuietly(transaction.getValue().producer) + ); } // ------------------- Logic for handling checkpoint flushing -------------------------- // @@ -691,7 +695,7 @@ protected FlinkKafkaProducer.KafkaTransactionState beginTransaction() throws Fli if (currentTransaction != null && currentTransaction.producer != null) { return new FlinkKafkaProducer.KafkaTransactionState(currentTransaction.producer); } - return new FlinkKafkaProducer.KafkaTransactionState(initProducer(true)); + return new FlinkKafkaProducer.KafkaTransactionState(initNonTransactionalProducer(true)); default: throw new UnsupportedOperationException("Not implemented semantic"); } @@ -714,73 +718,49 @@ protected void preCommit(FlinkKafkaProducer.KafkaTransactionState transaction) t @Override protected void commit(FlinkKafkaProducer.KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: + if (transaction.isTransactional()) { + try { transaction.producer.commitTransaction(); + } finally { recycleTransactionalProducer(transaction.producer); - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + } } } @Override protected void recoverAndCommit(FlinkKafkaProducer.KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - try (FlinkKafkaInternalProducer producer = + if (transaction.isTransactional()) { + try ( + FlinkKafkaInternalProducer producer = initTransactionalProducer(transaction.transactionalId, false)) { - producer.resumeTransaction(transaction.producerId, transaction.epoch); - producer.commitTransaction(); - } - catch (InvalidTxnStateException | ProducerFencedException ex) { - // That means we have committed this transaction before. - LOG.warn("Encountered error {} while recovering transaction {}. " + - "Presumably this transaction has been already committed before", - ex, - transaction); - } - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + producer.resumeTransaction(transaction.producerId, transaction.epoch); + producer.commitTransaction(); + } catch (InvalidTxnStateException | ProducerFencedException ex) { + // That means we have committed this transaction before. + LOG.warn("Encountered error {} while recovering transaction {}. " + + "Presumably this transaction has been already committed before", + ex, + transaction); + } } } @Override protected void abort(FlinkKafkaProducer.KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - transaction.producer.abortTransaction(); - recycleTransactionalProducer(transaction.producer); - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + if (transaction.isTransactional()) { + transaction.producer.abortTransaction(); + recycleTransactionalProducer(transaction.producer); } } @Override protected void recoverAndAbort(FlinkKafkaProducer.KafkaTransactionState transaction) { - switch (semantic) { - case EXACTLY_ONCE: - try (FlinkKafkaInternalProducer producer = + if (transaction.isTransactional()) { + try ( + FlinkKafkaInternalProducer producer = initTransactionalProducer(transaction.transactionalId, false)) { - producer.initTransactions(); - } - break; - case AT_LEAST_ONCE: - case NONE: - break; - default: - throw new UnsupportedOperationException("Not implemented semantic"); + producer.initTransactions(); + } } } @@ -895,7 +875,7 @@ protected void finishRecoveringContext() { LOG.info("Recovered transactionalIds {}", getUserContext().get().transactionalIds); } - protected FlinkKafkaInternalProducer createProducer() { + protected FlinkKafkaInternalProducer createProducer() { return new FlinkKafkaInternalProducer<>(this.producerConfig); } @@ -911,9 +891,7 @@ private void cleanUpUserContext() { private void resetAvailableTransactionalIdsPool(Collection transactionalIds) { availableTransactionalIds.clear(); - for (String transactionalId : transactionalIds) { - availableTransactionalIds.add(transactionalId); - } + availableTransactionalIds.addAll(transactionalIds); } // ----------------------------------- Utilities -------------------------- @@ -963,6 +941,11 @@ private FlinkKafkaInternalProducer initTransactionalProducer(Str return initProducer(registerMetrics); } + private FlinkKafkaInternalProducer initNonTransactionalProducer(boolean registerMetrics) { + producerConfig.remove("transactional.id"); + return initProducer(registerMetrics); + } + private FlinkKafkaInternalProducer initProducer(boolean registerMetrics) { FlinkKafkaInternalProducer producer = createProducer(); @@ -1081,7 +1064,7 @@ static class KafkaTransactionState { } KafkaTransactionState( - String transactionalId, + @Nullable String transactionalId, long producerId, short epoch, FlinkKafkaInternalProducer producer) { @@ -1091,6 +1074,10 @@ static class KafkaTransactionState { this.producer = producer; } + boolean isTransactional() { + return transactionalId != null; + } + @Override public String toString() { return String.format( diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactory.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactory.java index b0dfc54e67171..2b498674a89df 100644 --- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactory.java +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactory.java @@ -39,7 +39,7 @@ public class KafkaTableSourceSinkFactory extends KafkaTableSourceSinkFactoryBase @Override protected String kafkaVersion() { - return KafkaValidator.CONNECTOR_VERSION_VALUE_20; + return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL; } @Override diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java index b14b3e2524bfc..29f157ffb4fc3 100644 --- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java @@ -49,8 +49,8 @@ import static org.apache.flink.util.ExceptionUtils.findThrowable; import static org.apache.flink.util.Preconditions.checkState; import static org.hamcrest.Matchers.lessThan; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; /** * IT cases for the {@link FlinkKafkaProducer}. @@ -151,7 +151,7 @@ public void testRestoreToCheckpointAfterExceedingProducersPool() throws Exceptio testHarness2.open(); } - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42)); deleteTestTopic(topic); } catch (Exception ex) { @@ -163,7 +163,7 @@ public void testRestoreToCheckpointAfterExceedingProducersPool() throws Exceptio } @Test - public void testFlinkKafkaProducer10FailBeforeNotify() throws Exception { + public void testFlinkKafkaProducerFailBeforeNotify() throws Exception { String topic = "flink-kafka-producer-fail-before-notify"; OneInputStreamOperatorTestHarness testHarness = createTestHarness(topic); @@ -181,7 +181,7 @@ public void testFlinkKafkaProducer10FailBeforeNotify() throws Exception { try { testHarness.processElement(44, 4); testHarness.snapshot(2, 5); - assertFalse(true); + fail(); } catch (Exception ex) { // expected @@ -199,13 +199,13 @@ public void testFlinkKafkaProducer10FailBeforeNotify() throws Exception { testHarness.initializeState(snapshot); testHarness.close(); - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @Test - public void testFlinkKafkaProducer10FailTransactionCoordinatorBeforeNotify() throws Exception { + public void testFlinkKafkaProducerFailTransactionCoordinatorBeforeNotify() throws Exception { String topic = "flink-kafka-producer-fail-transaction-coordinator-before-notify"; Properties properties = createProperties(); @@ -248,7 +248,7 @@ public void testFlinkKafkaProducer10FailTransactionCoordinatorBeforeNotify() thr testHarness2.open(); } - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @@ -294,7 +294,7 @@ public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception { // - aborted transactions with records 44 and 45 // - committed transaction with record 46 // - pending transaction with record 47 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46)); testHarness.close(); deleteTestTopic(topic); @@ -343,7 +343,7 @@ public void testFailAndRecoverSameCheckpointTwice() throws Exception { //now we should have: // - records 42 and 43 in committed transactions // - aborted transactions with records 44 and 45 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43)); deleteTestTopic(topic); } @@ -395,7 +395,7 @@ public void testScaleDownBeforeFirstCheckpoint() throws Exception { // - records 42, 43, 44 and 45 in aborted transactions // - committed transaction with record 46 // - pending transaction with record 47 - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46)); postScaleDownOperator1.close(); // ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids. @@ -464,8 +464,7 @@ public void testScaleUpAfterScalingDown() throws Exception { createProperties(), topic, 0, - IntStream.range(0, parallelism1 + parallelism2 + parallelism3).boxed().collect(Collectors.toList()), - 30_000L); + IntStream.range(0, parallelism1 + parallelism2 + parallelism3).boxed().collect(Collectors.toList())); deleteTestTopic(topic); } @@ -537,7 +536,7 @@ public void testRecoverCommittedTransaction() throws Exception { testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0 testHarness.close(); - assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42), 30_000L); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42)); deleteTestTopic(topic); } @@ -564,6 +563,52 @@ public void testRunOutOfProducersInThePool() throws Exception { deleteTestTopic(topic); } + @Test + public void testMigrateFromAtLeastOnceToExactlyOnce() throws Exception { + String topic = "testMigrateFromAtLeastOnceToExactlyOnce"; + testRecoverWithChangeSemantics(topic, FlinkKafkaProducer.Semantic.AT_LEAST_ONCE, FlinkKafkaProducer.Semantic.EXACTLY_ONCE); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 44, 45)); + deleteTestTopic(topic); + } + + @Test + public void testMigrateFromAtExactlyOnceToAtLeastOnce() throws Exception { + String topic = "testMigrateFromExactlyOnceToAtLeastOnce"; + testRecoverWithChangeSemantics(topic, FlinkKafkaProducer.Semantic.EXACTLY_ONCE, FlinkKafkaProducer.Semantic.AT_LEAST_ONCE); + assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 45, 46, 47)); + deleteTestTopic(topic); + } + + private void testRecoverWithChangeSemantics( + String topic, + FlinkKafkaProducer.Semantic fromSemantic, + FlinkKafkaProducer.Semantic toSemantic) throws Exception { + OperatorSubtaskState producerSnapshot; + try (OneInputStreamOperatorTestHarness testHarness = createTestHarness(topic, fromSemantic)) { + testHarness.setup(); + testHarness.open(); + testHarness.processElement(42, 0); + testHarness.snapshot(0, 1); + testHarness.processElement(43, 2); + testHarness.notifyOfCompletedCheckpoint(0); + producerSnapshot = testHarness.snapshot(1, 3); + testHarness.processElement(44, 4); + } + + try (OneInputStreamOperatorTestHarness testHarness = createTestHarness(topic, toSemantic)) { + testHarness.setup(); + testHarness.initializeState(producerSnapshot); + testHarness.open(); + testHarness.processElement(45, 7); + testHarness.snapshot(2, 8); + testHarness.processElement(46, 9); + testHarness.notifyOfCompletedCheckpoint(2); + testHarness.processElement(47, 9); + } + } + + // ----------------------------------------------------------------------------------------------------------------- + // shut down a Kafka broker private void failBroker(int brokerId) { KafkaServer toShutDown = null; @@ -602,7 +647,13 @@ private void closeIgnoringProducerFenced(AutoCloseable autoCloseable) throws Exc } private OneInputStreamOperatorTestHarness createTestHarness(String topic) throws Exception { - return createTestHarness(topic, 1, 1, 0, FlinkKafkaProducer.Semantic.EXACTLY_ONCE); + return createTestHarness(topic, FlinkKafkaProducer.Semantic.EXACTLY_ONCE); + } + + private OneInputStreamOperatorTestHarness createTestHarness( + String topic, + FlinkKafkaProducer.Semantic semantic) throws Exception { + return createTestHarness(topic, 1, 1, 0, semantic); } private OneInputStreamOperatorTestHarness createTestHarness( diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTest.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTest.java index 5043880b21e14..4d843bcd82e05 100644 --- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTest.java +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTest.java @@ -40,7 +40,7 @@ public class KafkaTableSourceSinkFactoryTest extends KafkaTableSourceSinkFactory @Override protected String getKafkaVersion() { - return KafkaValidator.CONNECTOR_VERSION_VALUE_20; + return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL; } @Override diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..a7dcc98cbadd0 --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE @@ -0,0 +1,26 @@ +flink-connector-kinesis +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.amazonaws:aws-java-sdk-core:1.11.319 +- com.amazonaws:aws-java-sdk-kinesis:1.11.319 +- com.amazonaws:aws-java-sdk-sts:1.11.319 +- com.amazonaws:jmespath-java:1.11.319 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.google.protobuf:protobuf-java:2.6.1 + +This project bundles the following dependencies under the Amazon Softward License. +See bundled license files for details. + +- com.amazonaws:amazon-kinesis-client:1.9.0 +- com.amazonaws:amazon-kinesis-producer:0.12.9 + diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon new file mode 100644 index 0000000000000..e933f6953c1ca --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon @@ -0,0 +1,39 @@ +Amazon Software License + +This Amazon Software License (“License”) governs your use, reproduction, and distribution of the accompanying software as specified below. +1. Definitions + +“Licensor” means any person or entity that distributes its Work. + +“Software” means the original work of authorship made available under this License. + +“Work” means the Software and any additions to or derivative works of the Software that are made available under this License. + +The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. + +Works, including the Software, are “made available” under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. +2. License Grants + +2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. + +2.2 Patent Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer its Work, in whole or in part. The foregoing license applies only to the patent claims licensable by Licensor that would be infringed by Licensor’s Work (or portion thereof) individually and excluding any combinations with any other materials or technology. +3. Limitations + +3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work. + +3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself. + +3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc. + +3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1 and 2.2) will terminate immediately. + +3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License. + +3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1 and 2.2) will terminate immediately. +4. Disclaimer of Warranty. + +THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU. +5. Limitation of Liability. + +EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/flink-connectors/flink-connector-twitter/pom.xml b/flink-connectors/flink-connector-twitter/pom.xml index 27c7cbf6730ef..729c8c5808616 100644 --- a/flink-connectors/flink-connector-twitter/pom.xml +++ b/flink-connectors/flink-connector-twitter/pom.xml @@ -94,9 +94,6 @@ under the License. - - - diff --git a/flink-connectors/flink-connector-twitter/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-twitter/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..c4e49bc2270c3 --- /dev/null +++ b/flink-connectors/flink-connector-twitter/src/main/resources/META-INF/NOTICE @@ -0,0 +1,13 @@ +flink-connector-twitter +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:14.0.1 +- com.twitter:hbc-core:2.2.0 +- com.twitter:joauth:6.0.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml index bacc4ff7116be..c8c1007cb5b94 100644 --- a/flink-connectors/flink-hbase/pom.xml +++ b/flink-connectors/flink-hbase/pom.xml @@ -49,19 +49,6 @@ under the License. 1 - - org.apache.maven.plugins - maven-shade-plugin - - - - - - shade-flink - none - - - @@ -255,7 +242,7 @@ under the License. org.apache.hbase hbase-server ${hbase.version} - tests + test-jar test diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml index 3b5f78ddea9bb..74e715b4601ea 100644 --- a/flink-connectors/flink-hcatalog/pom.xml +++ b/flink-connectors/flink-hcatalog/pom.xml @@ -29,7 +29,7 @@ under the License. .. - flink-hcatalog + flink-hcatalog_${scala.binary.version} flink-hcatalog jar diff --git a/flink-connectors/flink-jdbc/pom.xml b/flink-connectors/flink-jdbc/pom.xml index 96c984f65beb8..9f34c4935dc25 100644 --- a/flink-connectors/flink-jdbc/pom.xml +++ b/flink-connectors/flink-jdbc/pom.xml @@ -30,7 +30,7 @@ under the License. .. - flink-jdbc + flink-jdbc_${scala.binary.version} flink-jdbc jar diff --git a/flink-container/docker/docker-entrypoint.sh b/flink-container/docker/docker-entrypoint.sh index 85ba8662bee2b..0bf7c04fa93c0 100755 --- a/flink-container/docker/docker-entrypoint.sh +++ b/flink-container/docker/docker-entrypoint.sh @@ -32,7 +32,6 @@ if [ "${CMD}" == "--help" -o "${CMD}" == "-h" ]; then exit 0 elif [ "${CMD}" == "${JOB_CLUSTER}" -o "${CMD}" == "${TASK_MANAGER}" ]; then echo "Starting the ${CMD}" - echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml if [ "${CMD}" == "${TASK_MANAGER}" ]; then exec $FLINK_HOME/bin/taskmanager.sh start-foreground "$@" diff --git a/flink-container/kubernetes/task-manager-deployment.yaml.template b/flink-container/kubernetes/task-manager-deployment.yaml.template index 55bd9a934c22f..f75b73234fa70 100644 --- a/flink-container/kubernetes/task-manager-deployment.yaml.template +++ b/flink-container/kubernetes/task-manager-deployment.yaml.template @@ -31,4 +31,9 @@ spec: containers: - name: flink-task-manager image: ${FLINK_IMAGE_NAME} - args: ["task-manager", "-Djobmanager.rpc.address=flink-job-cluster"] + args: ["task-manager", "-Djobmanager.rpc.address=flink-job-cluster", "-Dtaskmanager.host=$(K8S_POD_IP)"] + env: + - name: K8S_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/flink-contrib/docker-flink/docker-entrypoint.sh b/flink-contrib/docker-flink/docker-entrypoint.sh index 1edf99ca6ab9f..a4e279fad3bdc 100755 --- a/flink-contrib/docker-flink/docker-entrypoint.sh +++ b/flink-contrib/docker-flink/docker-entrypoint.sh @@ -29,7 +29,6 @@ elif [ "$1" == "jobmanager" ]; then echo "Starting Job Manager" sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: ${JOB_MANAGER_RPC_ADDRESS}/g" $FLINK_HOME/conf/flink-conf.yaml - echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml exec $FLINK_HOME/bin/jobmanager.sh start-foreground elif [ "$1" == "taskmanager" ]; then @@ -37,7 +36,6 @@ elif [ "$1" == "taskmanager" ]; then sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: $(grep -c ^processor /proc/cpuinfo)/g" $FLINK_HOME/conf/flink-conf.yaml echo "Starting Task Manager" - echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml exec $FLINK_HOME/bin/taskmanager.sh start-foreground fi diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml index a7d597a3ddfb9..c65832fee04b6 100644 --- a/flink-contrib/flink-storm-examples/pom.xml +++ b/flink-contrib/flink-storm-examples/pom.xml @@ -111,6 +111,14 @@ under the License. + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + org.apache.maven.plugins diff --git a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java index efbc4fac03906..32eed69e2cee0 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; @@ -361,10 +362,14 @@ public int getMaximumParallelism() { private static final class MaxDopVisitor implements Visitor> { + private final Set visitedOperators = new HashSet<>(); private int maxDop = -1; - + @Override public boolean preVisit(Operator visitable) { + if (!visitedOperators.add(visitable)) { + return false; + } this.maxDop = Math.max(this.maxDop, visitable.getParallelism()); return true; } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java index 55f3df7d31c0e..3a0b84c844987 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java @@ -83,7 +83,7 @@ public class CollectionExecutor { private final Map> aggregators; - private final ClassLoader classLoader; + private final ClassLoader userCodeClassLoader; private final ExecutionConfig executionConfig; @@ -99,7 +99,7 @@ public CollectionExecutor(ExecutionConfig executionConfig) { this.previousAggregates = new HashMap(); this.aggregators = new HashMap>(); this.cachedFiles = new HashMap>(); - this.classLoader = getClass().getClassLoader(); + this.userCodeClassLoader = Thread.currentThread().getContextClassLoader(); } // -------------------------------------------------------------------------------------------- @@ -191,8 +191,8 @@ private void executeDataSink(GenericDataSinkBase sink, int superStep) th MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) { - ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : - new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics); + ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : + new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); } else { ctx = null; } @@ -211,8 +211,8 @@ private List executeDataSource(GenericDataSourceBase source, in MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) { - ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : - new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics); + ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : + new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); } else { ctx = null; } @@ -237,8 +237,8 @@ private List executeUnaryOperator(SingleInputOperator op MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { - ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : - new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics); + ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : + new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); for (Map.Entry> bcInputs : operator.getBroadcastInputs().entrySet()) { List bcData = execute(bcInputs.getValue()); @@ -278,8 +278,8 @@ private List executeBinaryOperator(DualInputOperator> bcInputs : operator.getBroadcastInputs().entrySet()) { List bcData = execute(bcInputs.getValue()); diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java index ddb0b87e52574..5e9498ecf1ea8 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java @@ -29,9 +29,8 @@ * This interface describes the methods that are required for a data type to be handled by the Flink * runtime. Specifically, this interface contains the serialization and copying methods. * - *

The methods in this class are assumed to be stateless, such that it is effectively thread safe. Stateful - * implementations of the methods may lead to unpredictable side effects and will compromise both stability and - * correctness of the program. + *

The methods in this class are not necessarily thread safe. To avoid unpredictable side effects, + * it is recommended to call {@code duplicate()} method and use one serializer instance per thread. * *

Upgrading TypeSerializers to the new TypeSerializerSnapshot model * diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java index 2caf9fa18c617..5f89d94973f81 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; import java.io.IOException; import java.util.List; @@ -48,7 +49,7 @@ public ListSerializerSnapshot() {} * Constructor to create the snapshot for writing. */ public ListSerializerSnapshot(TypeSerializer elementSerializer) { - this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(elementSerializer); + this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(elementSerializer)); } @Override diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java index 92d17996ea5c1..be2e4b0cbe6a0 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; import java.io.IOException; import java.util.Map; @@ -48,6 +49,8 @@ public MapSerializerSnapshot() {} * Constructor to create the snapshot for writing. */ public MapSerializerSnapshot(TypeSerializer keySerializer, TypeSerializer valueSerializer) { + Preconditions.checkNotNull(keySerializer); + Preconditions.checkNotNull(valueSerializer); this.nestedKeyValueSerializerSnapshot = new CompositeSerializerSnapshot(keySerializer, valueSerializer); } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java index a4d3839f66f10..f1dd8fcbde952 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java @@ -18,29 +18,17 @@ package org.apache.flink.api.java.typeutils.runtime; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Objects; - import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; @@ -52,6 +40,18 @@ import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.util.Preconditions; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; + import static org.apache.flink.util.Preconditions.checkNotNull; @Internal @@ -181,11 +181,13 @@ public PojoSerializer duplicate() { } } - if (stateful) { - return new PojoSerializer(clazz, duplicateFieldSerializers, fields, executionConfig); - } else { - return this; + if (!stateful) { + // as a small memory optimization, we can share the same object between instances + duplicateFieldSerializers = fieldSerializers; } + + // we must create a new instance, otherwise the subclassSerializerCache can create concurrency problems + return new PojoSerializer<>(clazz, duplicateFieldSerializers, fields, executionConfig); } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/Tuple0Serializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/Tuple0Serializer.java index b96126d2f64b5..e173d7b3bac19 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/Tuple0Serializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/Tuple0Serializer.java @@ -19,6 +19,7 @@ import org.apache.flink.api.java.tuple.Tuple0; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; @Internal public class Tuple0Serializer extends TupleSerializer { @@ -72,6 +73,7 @@ public int getLength() { @Override public void serialize(Tuple0 record, DataOutputView target) throws IOException { + Preconditions.checkNotNull(record); target.writeByte(42); } diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java index 6557a9f7dd401..2b4bba66c5e9e 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java @@ -71,17 +71,31 @@ public class CheckpointingOptions { /** * This option configures local recovery for this state backend. By default, local recovery is deactivated. + * + *

Local recovery currently only covers keyed state backends. + * Currently, MemoryStateBackend does not support local recovery and ignore + * this option. */ public static final ConfigOption LOCAL_RECOVERY = ConfigOptions - .key("state.backend.local-recovery") - .defaultValue(false); + .key("state.backend.local-recovery") + .defaultValue(false) + .withDescription("This option configures local recovery for this state backend. By default, local recovery is " + + "deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does " + + "not support local recovery and ignore this option."); /** * The config parameter defining the root directories for storing file-based state for local recovery. + * + *

Local recovery currently only covers keyed state backends. + * Currently, MemoryStateBackend does not support local recovery and ignore + * this option. */ public static final ConfigOption LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS = ConfigOptions - .key("taskmanager.state.local.root-dirs") - .noDefaultValue(); + .key("taskmanager.state.local.root-dirs") + .noDefaultValue() + .withDescription("The config parameter defining the root directories for storing file-based state for local " + + "recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does " + + "not support local recovery and ignore this option"); // ------------------------------------------------------------------------ // Options specific to the file-system-based state backends diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java new file mode 100644 index 0000000000000..e67dd073aa86a --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.configuration; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * Options which control the cluster behaviour. + */ +@PublicEvolving +public class ClusterOptions { + + public static final ConfigOption CLUSTER_SERVICES_SHUTDOWN_TIMEOUT = ConfigOptions + .key("cluster.services.shutdown-timeout") + .defaultValue(30000L) + .withDescription("The shutdown timeout for cluster services like executors in milliseconds."); +} diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java index 1666f213d182e..cb2dbe6163189 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java @@ -106,6 +106,7 @@ public class JobManagerOptions { /** * This option specifies the failover strategy, i.e. how the job computation recovers from task failures. */ + @Documentation.ExcludeFromDocumentation("The failover strategy feature is highly experimental.") public static final ConfigOption EXECUTION_FAILOVER_STRATEGY = key("jobmanager.execution.failover-strategy") .defaultValue("full") diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index 98fa47f669704..61331b28d57b7 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -19,16 +19,23 @@ package org.apache.flink.configuration; import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.annotation.docs.ConfigGroup; +import org.apache.flink.annotation.docs.ConfigGroups; import org.apache.flink.annotation.docs.Documentation; +import org.apache.flink.configuration.description.Description; import static org.apache.flink.configuration.ConfigOptions.key; +import static org.apache.flink.configuration.description.TextElement.code; /** * The set of configuration options relating to TaskManager and Task settings. */ @PublicEvolving +@ConfigGroups(groups = @ConfigGroup(name = "TaskManagerMemory", keyPrefix = "taskmanager.memory")) public class TaskManagerOptions { + private static final String MANAGED_MEMORY_PRE_ALLOCATE_KEY = "taskmanager.memory.preallocate"; + // ------------------------------------------------------------------------ // General TaskManager Options // ------------------------------------------------------------------------ @@ -204,8 +211,10 @@ public class TaskManagerOptions { public static final ConfigOption MANAGED_MEMORY_SIZE = key("taskmanager.memory.size") .defaultValue("0") - .withDescription("Amount of memory to be allocated by the task manager's memory manager." + - " If not set, a relative fraction will be allocated."); + .withDescription("The amount of memory (in megabytes) that the task manager reserves on-heap or off-heap" + + " (depending on taskmanager.memory.off-heap) for sorting, hash tables, and caching of intermediate" + + " results. If unspecified, the memory manager will take a fixed ratio with respect to the size of" + + " the task manager JVM as specified by taskmanager.memory.fraction."); /** * Fraction of free memory allocated by the memory manager if {@link #MANAGED_MEMORY_SIZE} is @@ -214,12 +223,15 @@ public class TaskManagerOptions { public static final ConfigOption MANAGED_MEMORY_FRACTION = key("taskmanager.memory.fraction") .defaultValue(0.7f) - .withDescription("The relative amount of memory (after subtracting the amount of memory used by network" + - " buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results." + - " For example, a value of `0.8` means that a task manager reserves 80% of its memory" + - " for internal data buffers, leaving 20% of free memory for the task manager's heap for objects" + - " created by user-defined functions. This parameter is only evaluated, if " + MANAGED_MEMORY_SIZE.key() + - " is not set."); + .withDescription(Description.builder() + .text("The relative amount of memory (after subtracting the amount of memory used by network" + + " buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results." + + " For example, a value of %s means that a task manager reserves 80% of its memory" + + " (on-heap or off-heap depending on taskmanager.memory.off-heap)" + + " for internal data buffers, leaving 20% of free memory for the task manager's heap for objects" + + " created by user-defined functions. This parameter is only evaluated, if " + MANAGED_MEMORY_SIZE.key() + + " is not set.", code("0.8")) + .build()); /** * Memory allocation method (JVM heap or off-heap), used for managed memory of the TaskManager @@ -228,16 +240,27 @@ public class TaskManagerOptions { public static final ConfigOption MEMORY_OFF_HEAP = key("taskmanager.memory.off-heap") .defaultValue(false) - .withDescription("Memory allocation method (JVM heap or off-heap), used for managed memory of the" + - " TaskManager as well as the network buffers."); + .withDescription(Description.builder() + .text("Memory allocation method (JVM heap or off-heap), used for managed memory of the" + + " TaskManager. For setups with larger quantities of memory, this can" + + " improve the efficiency of the operations performed on the memory.") + .linebreak() + .text("When set to true, then it is advised that %s is also set to true.", code(MANAGED_MEMORY_PRE_ALLOCATE_KEY)) + .build()); /** * Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting. */ public static final ConfigOption MANAGED_MEMORY_PRE_ALLOCATE = - key("taskmanager.memory.preallocate") + key(MANAGED_MEMORY_PRE_ALLOCATE_KEY) .defaultValue(false) - .withDescription("Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting."); + .withDescription(Description.builder() + .text("Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting." + + " When %s is set to true, then it is advised that this configuration is also" + + " set to true. If this configuration is set to false cleaning up of the allocated off-heap memory" + + " happens only when the configured JVM parameter MaxDirectMemorySize is reached by triggering a full" + + " GC. For streaming setups is is highly recommended to set this value to false as the core state" + + " backends currently do not use the managed memory.", code(MEMORY_OFF_HEAP.key())).build()); // ------------------------------------------------------------------------ // Network Options @@ -316,7 +339,7 @@ public class TaskManagerOptions { key("taskmanager.network.request-backoff.initial") .defaultValue(100) .withDeprecatedKeys("taskmanager.net.request-backoff.initial") - .withDescription("Minimum backoff for partition requests of input channels."); + .withDescription("Minimum backoff in milliseconds for partition requests of input channels."); /** * Maximum backoff for partition requests of input channels. @@ -325,7 +348,7 @@ public class TaskManagerOptions { key("taskmanager.network.request-backoff.max") .defaultValue(10000) .withDeprecatedKeys("taskmanager.net.request-backoff.max") - .withDescription("Maximum backoff for partition requests of input channels."); + .withDescription("Maximum backoff in milliseconds for partition requests of input channels."); /** * Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue diff --git a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java index a47530315c5c2..53f540ed63f1f 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java @@ -32,7 +32,7 @@ protected void formatLink(StringBuilder state, String link, String description) @Override protected void formatLineBreak(StringBuilder state) { - state.append("
"); + state.append("
"); } @Override diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java b/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java index e5bfdb84a02f2..7d54b11bbbc82 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java @@ -121,6 +121,34 @@ public interface RecoverableWriter { */ RecoverableFsDataOutputStream recover(ResumeRecoverable resumable) throws IOException; + /** + * Marks if the writer requires to do any additional cleanup/freeing of resources occupied + * as part of a {@link ResumeRecoverable}, e.g. temporarily files created or objects uploaded + * to external systems. + * + *

In case cleanup is required, then {@link #cleanupRecoverableState(ResumeRecoverable)} should + * be called. + * + * @return {@code true} if cleanup is required, {@code false} otherwise. + */ + boolean requiresCleanupOfRecoverableState(); + + /** + * Frees up any resources that were previously occupied in order to be able to + * recover from a (potential) failure. These can be temporary files that were written + * to the filesystem or objects that were uploaded to S3. + * + *

NOTE: This operation should not throw an exception if the resumable has already + * been cleaned up and the resources have been freed. But the contract is that it will throw + * an {@link UnsupportedOperationException} if it is called for a {@code RecoverableWriter} + * whose {@link #requiresCleanupOfRecoverableState()} returns {@code false}. + * + * @param resumable The {@link ResumeRecoverable} whose state we want to clean-up. + * @return {@code true} if the resources were successfully freed, {@code false} otherwise + * (e.g. the file to be deleted was not there for any reason - already deleted or never created). + */ + boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException; + /** * Recovers a recoverable stream consistently at the point indicated by the given CommitRecoverable * for finalizing and committing. This will publish the target file with exactly the data diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java index a2f0485a003b6..a43e0b6b6bf4e 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java @@ -70,6 +70,16 @@ public RecoverableFsDataOutputStream recover(ResumeRecoverable recoverable) thro } } + @Override + public boolean requiresCleanupOfRecoverableState() { + return false; + } + + @Override + public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public Committer recoverForCommit(CommitRecoverable recoverable) throws IOException { if (recoverable instanceof LocalRecoverable) { diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java index eab8f4ccde677..644289133b283 100644 --- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java +++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java @@ -43,6 +43,7 @@ import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; +import java.lang.reflect.Proxy; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -91,6 +92,40 @@ protected Class resolveClass(ObjectStreamClass desc) throws IOException, Clas return super.resolveClass(desc); } + @Override + protected Class resolveProxyClass(String[] interfaces) throws IOException, ClassNotFoundException { + if (classLoader != null) { + ClassLoader nonPublicLoader = null; + boolean hasNonPublicInterface = false; + + // define proxy in class loader of non-public interface(s), if any + Class[] classObjs = new Class[interfaces.length]; + for (int i = 0; i < interfaces.length; i++) { + Class cl = Class.forName(interfaces[i], false, classLoader); + if ((cl.getModifiers() & Modifier.PUBLIC) == 0) { + if (hasNonPublicInterface) { + if (nonPublicLoader != cl.getClassLoader()) { + throw new IllegalAccessError( + "conflicting non-public interface class loaders"); + } + } else { + nonPublicLoader = cl.getClassLoader(); + hasNonPublicInterface = true; + } + } + classObjs[i] = cl; + } + try { + return Proxy.getProxyClass( + hasNonPublicInterface ? nonPublicLoader : classLoader, classObjs); + } catch (IllegalArgumentException e) { + throw new ClassNotFoundException(null, e); + } + } + + return super.resolveProxyClass(interfaces); + } + // ------------------------------------------------ private static final HashMap> primitiveClasses = new HashMap<>(9); @@ -210,11 +245,12 @@ protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFo final Class localClass = resolveClass(streamClassDescriptor); final String name = localClass.getName(); - if (scalaSerializerClassnames.contains(name) || scalaTypes.contains(name) || isAnonymousClass(localClass)) { + if (scalaSerializerClassnames.contains(name) || scalaTypes.contains(name) || isAnonymousClass(localClass) + || isOldAvroSerializer(name, streamClassDescriptor.getSerialVersionUID())) { final ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass); if (localClassDescriptor != null && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) { - LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.", + LOG.warn("Ignoring serialVersionUID mismatch for class {}; was {}, now {}.", streamClassDescriptor.getName(), streamClassDescriptor.getSerialVersionUID(), localClassDescriptor.getSerialVersionUID()); streamClassDescriptor = localClassDescriptor; @@ -223,6 +259,7 @@ protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFo return streamClassDescriptor; } + } private static boolean isAnonymousClass(Class clazz) { @@ -242,6 +279,11 @@ private static boolean isAnonymousClass(Class clazz) { } } + private static boolean isOldAvroSerializer(String name, long serialVersionUID) { + // please see FLINK-11436 for details on why we need to ignore serial version UID here for the AvroSerializer + return (serialVersionUID == 1) && "org.apache.flink.formats.avro.typeutils.AvroSerializer".equals(name); + } + /** * A mapping between the full path of a deprecated serializer and its equivalent. * These mappings are hardcoded and fixed. diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeSerializerTest.java index fc5c241c20566..ab3e6f3a84c49 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeSerializerTest.java @@ -37,8 +37,6 @@ import java.util.function.IntFunction; import java.util.stream.IntStream; -import static org.junit.Assert.assertEquals; - /** Test suite for {@link CompositeSerializer}. */ public class CompositeSerializerTest { private static final ExecutionConfig execConf = new ExecutionConfig(); @@ -205,9 +203,5 @@ private static class CompositeSerializerTestInstance extends SerializerTestInsta private static Class> getCls(List instance) { return TypeExtractor.getForObject(instance).getTypeClass(); } - - protected void deepEquals(String message, List should, List is) { - assertEquals(message, should, is); - } } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java index a4908f95e90c2..f57b99b17816a 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java @@ -18,32 +18,38 @@ package org.apache.flink.api.common.typeutils; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.Arrays; - import org.apache.flink.api.java.typeutils.runtime.NullableSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.testutils.CustomEqualityMatcher; +import org.apache.flink.testutils.DeeplyEqualsChecker; import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.TestLogger; -import org.junit.Assert; import org.apache.commons.lang3.SerializationException; import org.apache.commons.lang3.SerializationUtils; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; +import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Abstract test base for serializers. * @@ -54,6 +60,16 @@ */ public abstract class SerializerTestBase extends TestLogger { + private final DeeplyEqualsChecker checker; + + protected SerializerTestBase() { + this.checker = new DeeplyEqualsChecker(); + } + + protected SerializerTestBase(DeeplyEqualsChecker checker) { + this.checker = checker; + } + protected abstract TypeSerializer createSerializer(); /** @@ -436,52 +452,42 @@ public void testNullability() { } } - // -------------------------------------------------------------------------------------------- + @Test + public void testDuplicate() throws Exception { + final int numThreads = 10; + final TypeSerializer serializer = getSerializer(); + final CyclicBarrier startLatch = new CyclicBarrier(numThreads); + final List> concurrentRunners = new ArrayList<>(numThreads); + Assert.assertEquals(serializer, serializer.duplicate()); - protected void deepEquals(String message, T should, T is) { - Assert.assertTrue((should == null && is == null) || (should != null && is != null)); - if (should == null) { - return; - } - if (should.getClass().isArray()) { - if (should instanceof boolean[]) { - Assert.assertTrue(message, Arrays.equals((boolean[]) should, (boolean[]) is)); - } - else if (should instanceof byte[]) { - assertArrayEquals(message, (byte[]) should, (byte[]) is); - } - else if (should instanceof short[]) { - assertArrayEquals(message, (short[]) should, (short[]) is); - } - else if (should instanceof int[]) { - assertArrayEquals(message, (int[]) should, (int[]) is); - } - else if (should instanceof long[]) { - assertArrayEquals(message, (long[]) should, (long[]) is); - } - else if (should instanceof float[]) { - assertArrayEquals(message, (float[]) should, (float[]) is, 0.0f); - } - else if (should instanceof double[]) { - assertArrayEquals(message, (double[]) should, (double[]) is, 0.0); - } - else if (should instanceof char[]) { - assertArrayEquals(message, (char[]) should, (char[]) is); - } - else { - assertArrayEquals(message, (Object[]) should, (Object[]) is); - } - } - else if (should instanceof Throwable) { - assertEquals(((Throwable)should).getMessage(), ((Throwable)is).getMessage()); + T[] testData = getData(); + + for (int i = 0; i < numThreads; ++i) { + SerializerRunner runner = new SerializerRunner<>( + startLatch, + serializer.duplicate(), + testData, + 120L, + checker); + + runner.start(); + concurrentRunners.add(runner); } - else { - assertEquals(message, should, is); + + for (SerializerRunner concurrentRunner : concurrentRunners) { + concurrentRunner.join(); + concurrentRunner.checkResult(); } } // -------------------------------------------------------------------------------------------- + private void deepEquals(String message, T should, T is) { + assertThat(message, is, CustomEqualityMatcher.deeplyEquals(should).withChecker(checker)); + } + + // -------------------------------------------------------------------------------------------- + protected TypeSerializer getSerializer() { TypeSerializer serializer = createSerializer(); if (serializer == null) { @@ -526,6 +532,76 @@ public void write(DataInputView source, int numBytes) throws IOException { } } + /** + * Runner to test serializer duplication via concurrency. + * @param type of the test elements. + */ + static class SerializerRunner extends Thread { + final CyclicBarrier allReadyBarrier; + final TypeSerializer serializer; + final T[] testData; + final long durationLimitMillis; + Throwable failure; + final DeeplyEqualsChecker checker; + + SerializerRunner( + CyclicBarrier allReadyBarrier, + TypeSerializer serializer, + T[] testData, + long testTargetDurationMillis, DeeplyEqualsChecker checker) { + + this.allReadyBarrier = allReadyBarrier; + this.serializer = serializer; + this.testData = testData; + this.durationLimitMillis = testTargetDurationMillis; + this.checker = checker; + this.failure = null; + } + + @Override + public void run() { + DataInputDeserializer dataInputDeserializer = new DataInputDeserializer(); + DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128); + try { + allReadyBarrier.await(); + final long endTimeNanos = System.nanoTime() + durationLimitMillis * 1_000_000L; + while (true) { + for (T testItem : testData) { + serializer.serialize(testItem, dataOutputSerializer); + dataInputDeserializer.setBuffer( + dataOutputSerializer.getSharedBuffer(), + 0, + dataOutputSerializer.length()); + T serdeTestItem = serializer.deserialize(dataInputDeserializer); + T copySerdeTestItem = serializer.copy(serdeTestItem); + dataOutputSerializer.clear(); + + assertThat( + "Serialization/Deserialization cycle resulted in an object that are not equal to the original.", + copySerdeTestItem, + CustomEqualityMatcher.deeplyEquals(testItem).withChecker(checker)); + + // try to enforce some upper bound to the test time + if (System.nanoTime() >= endTimeNanos) { + return; + } + } + } + } catch (Throwable ex) { + failure = ex; + } + } + + void checkResult() throws Exception { + if (failure != null) { + if (failure instanceof AssertionError) { + throw (AssertionError) failure; + } else { + throw (Exception) failure; + } + } + } + } private static final class TestInputView extends DataInputStream implements DataInputView { diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestInstance.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestInstance.java index a7207582ff4b8..ec25e9e27287d 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestInstance.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestInstance.java @@ -19,6 +19,13 @@ package org.apache.flink.api.common.typeutils; +import org.apache.flink.testutils.DeeplyEqualsChecker; + +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + public class SerializerTestInstance extends SerializerTestBase { private final TypeSerializer serializer; @@ -33,14 +40,25 @@ public class SerializerTestInstance extends SerializerTestBase { @SafeVarargs public SerializerTestInstance(TypeSerializer serializer, Class typeClass, int length, T... testData) { + this(new DeeplyEqualsChecker(), serializer, typeClass, length, testData); + } + + @SafeVarargs + public SerializerTestInstance( + DeeplyEqualsChecker checker, + TypeSerializer serializer, + Class typeClass, + int length, + T... testData) { + super(checker); this.serializer = serializer; this.typeClass = typeClass; this.length = length; this.testData = testData; } - + // -------------------------------------------------------------------------------------------- - + @Override protected TypeSerializer createSerializer() { return this.serializer; @@ -60,21 +78,26 @@ protected Class getTypeClass() { protected T[] getTestData() { return this.testData; } - + // -------------------------------------------------------------------------------------------- - + public void testAll() { - testInstantiate(); - testGetLength(); - testCopy(); - testCopyIntoNewElements(); - testCopyIntoReusedElements(); - testSerializeIndividually(); - testSerializeIndividuallyReusingValues(); - testSerializeAsSequenceNoReuse(); - testSerializeAsSequenceReusingValues(); - testSerializedCopyIndividually(); - testSerializedCopyAsSequence(); - testSerializabilityAndEquals(); + for (Method method : SerializerTestBase.class.getMethods()) { + if (method.getAnnotation(Test.class) == null) { + continue; + } + try { + method.invoke(this); + } catch (IllegalAccessException e) { + throw new RuntimeException("Unable to invoke test " + method.getName(), e); + } catch (InvocationTargetException e) { + sneakyThrow(e.getCause()); + } + } + } + + @SuppressWarnings("unchecked") + private static void sneakyThrow(Throwable e) throws E { + throw (E) e; } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java index e906f62f65bf6..fb11945305721 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java @@ -23,21 +23,17 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.testutils.ClassLoaderUtils; import org.apache.flink.util.TestLogger; + import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; public class EnumSerializerUpgradeTest extends TestLogger { @@ -87,7 +83,7 @@ public void checkDifferentFieldOrder() throws Exception { private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB) throws IOException, ClassNotFoundException { - ClassLoader classLoader = compileAndLoadEnum( + ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava( temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA); EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME)); @@ -103,7 +99,7 @@ private static TypeSerializerSchemaCompatibility checkCompatibility(String enumS snapshotBytes = outBuffer.toByteArray(); } - ClassLoader classLoader2 = compileAndLoadEnum( + ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava( temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB); TypeSerializerSnapshot restoredSnapshot; @@ -118,29 +114,4 @@ private static TypeSerializerSchemaCompatibility checkCompatibility(String enumS EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME)); return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2); } - - private static ClassLoader compileAndLoadEnum(File root, String filename, String source) throws IOException { - File file = writeSourceFile(root, filename, source); - - compileClass(file); - - return new URLClassLoader( - new URL[]{root.toURI().toURL()}, - Thread.currentThread().getContextClassLoader()); - } - - private static File writeSourceFile(File root, String filename, String source) throws IOException { - File file = new File(root, filename); - FileWriter fileWriter = new FileWriter(file); - - fileWriter.write(source); - fileWriter.close(); - - return file; - } - - private static int compileClass(File sourceFile) { - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - return compiler.run(null, null, null, "-proc:none", sourceFile.getPath()); - } } diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java index d08d68a6678bc..f94943183839d 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java @@ -27,12 +27,11 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.types.Row; + import org.junit.Test; import java.io.Serializable; -import static org.junit.Assert.assertEquals; - public class RowSerializerTest { @Test @@ -150,17 +149,6 @@ private class RowSerializerTestInstance extends SerializerTestInstance { Row... testData) { super(serializer, Row.class, -1, testData); } - - @Override - protected void deepEquals(String message, Row should, Row is) { - int arity = should.getArity(); - assertEquals(message, arity, is.getArity()); - for (int i = 0; i < arity; i++) { - Object copiedValue = should.getField(i); - Object element = is.getField(i); - assertEquals(message, element, copiedValue); - } - } } public static class MyPojo implements Serializable, Comparable { diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java index 13f91b048c617..f6cf2def30233 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java @@ -23,6 +23,7 @@ import java.util.Random; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeutils.SerializerTestInstance; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple0; @@ -225,7 +226,7 @@ private void runTests(int length, T... instances) { if(tupleClass == Tuple0.class) { length = 1; } - TupleSerializerTestInstance test = new TupleSerializerTestInstance(serializer, tupleClass, length, instances); + SerializerTestInstance test = new SerializerTestInstance<>(serializer, tupleClass, length, instances); test.testAll(); } catch (Exception e) { diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTestInstance.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTestInstance.java deleted file mode 100644 index a196984d4e513..0000000000000 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTestInstance.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.flink.api.java.typeutils.runtime; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; - -import org.apache.flink.api.common.typeutils.SerializerTestInstance; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.tuple.Tuple; -import org.junit.Assert; - -public class TupleSerializerTestInstance extends SerializerTestInstance { - - public TupleSerializerTestInstance(TypeSerializer serializer, Class typeClass, int length, T[] testData) { - super(serializer, typeClass, length, testData); - } - - protected void deepEquals(String message, T shouldTuple, T isTuple) { - Assert.assertEquals(shouldTuple.getArity(), isTuple.getArity()); - - for (int i = 0; i < shouldTuple.getArity(); i++) { - Object should = shouldTuple.getField(i); - Object is = isTuple.getField(i); - - if (should.getClass().isArray()) { - if (should instanceof boolean[]) { - Assert.assertTrue(message, Arrays.equals((boolean[]) should, (boolean[]) is)); - } - else if (should instanceof byte[]) { - assertArrayEquals(message, (byte[]) should, (byte[]) is); - } - else if (should instanceof short[]) { - assertArrayEquals(message, (short[]) should, (short[]) is); - } - else if (should instanceof int[]) { - assertArrayEquals(message, (int[]) should, (int[]) is); - } - else if (should instanceof long[]) { - assertArrayEquals(message, (long[]) should, (long[]) is); - } - else if (should instanceof float[]) { - assertArrayEquals(message, (float[]) should, (float[]) is, 0.0f); - } - else if (should instanceof double[]) { - assertArrayEquals(message, (double[]) should, (double[]) is, 0.0); - } - else if (should instanceof char[]) { - assertArrayEquals(message, (char[]) should, (char[]) is); - } - else { - assertArrayEquals(message, (Object[]) should, (Object[]) is); - } - } - else { - assertEquals(message, should, is); - } - } - } -} diff --git a/flink-core/src/test/java/org/apache/flink/configuration/description/DescriptionHtmlTest.java b/flink-core/src/test/java/org/apache/flink/configuration/description/DescriptionHtmlTest.java index fcdb15583d647..bca827257efd1 100644 --- a/flink-core/src/test/java/org/apache/flink/configuration/description/DescriptionHtmlTest.java +++ b/flink-core/src/test/java/org/apache/flink/configuration/description/DescriptionHtmlTest.java @@ -93,7 +93,7 @@ public void testDescriptionWithLineBreak() { String formattedDescription = new HtmlFormatter().format(description); assertEquals( - "This is first line.
This is second line.", + "This is first line.
This is second line.", formattedDescription); } diff --git a/flink-core/src/test/java/org/apache/flink/testutils/ClassLoaderUtils.java b/flink-core/src/test/java/org/apache/flink/testutils/ClassLoaderUtils.java new file mode 100644 index 0000000000000..b10f2e175b83f --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/testutils/ClassLoaderUtils.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.testutils; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashMap; +import java.util.Map; + +/** + * Utilities to create class loaders. + */ +public class ClassLoaderUtils { + public static URLClassLoader compileAndLoadJava(File root, String filename, String source) throws + IOException { + return withRoot(root) + .addClass(filename.replaceAll("\\.java", ""), source) + .build(); + } + + private static URLClassLoader createClassLoader(File root) throws MalformedURLException { + return new URLClassLoader( + new URL[]{root.toURI().toURL()}, + Thread.currentThread().getContextClassLoader()); + } + + private static void writeAndCompile(File root, String filename, String source) throws IOException { + File file = writeSourceFile(root, filename, source); + + compileClass(file); + } + + private static File writeSourceFile(File root, String filename, String source) throws IOException { + File file = new File(root, filename); + FileWriter fileWriter = new FileWriter(file); + + fileWriter.write(source); + fileWriter.close(); + + return file; + } + + public static ClassLoaderBuilder withRoot(File root) { + return new ClassLoaderBuilder(root); + } + + private static int compileClass(File sourceFile) { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + return compiler.run(null, null, null, "-proc:none", sourceFile.getPath()); + } + + public static class ClassLoaderBuilder { + + private final File root; + private final Map classes; + + private ClassLoaderBuilder(File root) { + this.root = root; + this.classes = new HashMap<>(); + } + + public ClassLoaderBuilder addClass(String className, String source) { + String oldValue = classes.putIfAbsent(className, source); + + if (oldValue != null) { + throw new RuntimeException(String.format("Class with name %s already registered.", className)); + } + + return this; + } + + public URLClassLoader build() throws IOException { + for (Map.Entry classInfo : classes.entrySet()) { + writeAndCompile(root, createFileName(classInfo.getKey()), classInfo.getValue()); + } + + return createClassLoader(root); + } + + private String createFileName(String className) { + return className + ".java"; + } + } +} diff --git a/flink-core/src/test/java/org/apache/flink/testutils/CustomEqualityMatcher.java b/flink-core/src/test/java/org/apache/flink/testutils/CustomEqualityMatcher.java new file mode 100644 index 0000000000000..e80666c6222cb --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/testutils/CustomEqualityMatcher.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.testutils; + +import org.apache.flink.api.java.tuple.Tuple; + +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; + +import java.util.Arrays; + +public class CustomEqualityMatcher extends BaseMatcher { + + private final Object wanted; + + private final DeeplyEqualsChecker checker; + + private CustomEqualityMatcher(Object wanted, DeeplyEqualsChecker checker) { + this.wanted = wanted; + this.checker = checker; + } + + /** + * This matcher performs similar comparison to {@link org.hamcrest.core.IsEqual}, which resembles + * {@link java.util.Objects#deepEquals(Object, Object)} logic. The only difference here is that {@link Tuple}s + * are treated similarly to arrays. + * + *

This means that if we compare two Tuples that contain arrays, those arrays will + * be compared with {@link Arrays#deepEquals(Object[], Object[])} rather than with reference comparison. + * + * @param item expected value + */ + public static CustomEqualityMatcher deeplyEquals(Object item) { + return new CustomEqualityMatcher(item, new DeeplyEqualsChecker()); + } + + /** + * Performs assertions with this customly configured {@link DeeplyEqualsChecker}. It might have some additional + * rules applied. + */ + public CustomEqualityMatcher withChecker(DeeplyEqualsChecker checker) { + return new CustomEqualityMatcher(wanted, checker); + } + + @Override + public boolean matches(Object item) { + return checker.deepEquals(item, wanted); + } + + @Override + public void describeTo(Description description) { + description.appendValue(wanted); + } +} diff --git a/flink-core/src/test/java/org/apache/flink/testutils/DeeplyEqualsChecker.java b/flink-core/src/test/java/org/apache/flink/testutils/DeeplyEqualsChecker.java new file mode 100644 index 0000000000000..00119f161153a --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/testutils/DeeplyEqualsChecker.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.testutils; + +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; + +/** + * Deep equality checker for tests. It performs deep checks for objects which have no proper deepEquals methods like: + *

    + *
  • {@link Tuple}s
    • + *
  • {@link Iterable}s
    • + *
  • Java arrays
    • + *
  • {@link Row}
    • + *
  • {@link Throwable}
    • + *
+ * + *

One can also provide custom check for additional categories of objects with + * {@link DeeplyEqualsChecker#withCustomCheck(BiFunction, CustomEqualityChecker)}. This is used e.g. in scala's tests. + */ +public class DeeplyEqualsChecker { + + /** + * Checker that compares o1 and o2 objects if they are deeply equal. + * + *

NOTE: All nested comparisons should be done through checker. + */ + public interface CustomEqualityChecker { + boolean check(Object o1, Object o2, DeeplyEqualsChecker checker); + } + + private final List, CustomEqualityChecker>> customCheckers + = new ArrayList<>(); + + /** + * Adds custom check. Those check are always performed first, only after that it fallbacks to default checks. + * + * @param shouldCheck function to evaluate if the objects should be compared with comparator + * @param comparator to perform equality comparison if the shouldCheck passed + * @return checker with added custom checks + */ + public DeeplyEqualsChecker withCustomCheck( + BiFunction shouldCheck, + CustomEqualityChecker comparator) { + customCheckers.add(Tuple2.of(shouldCheck, comparator)); + return this; + } + + public boolean deepEquals(Object o1, Object o2) { + if (o1 == o2) { + return true; + } else if (o1 == null || o2 == null) { + return false; + } else { + return customCheck(o1, o2).orElseGet(() -> deepEquals0(o1, o2)); + } + } + + private Optional customCheck(Object o1, Object o2) { + return customCheckers.stream() + .filter(checker -> checker.f0.apply(o1, o2)) + .findAny() + .map(checker -> checker.f1.check(o1, o2, this)); + } + + private boolean deepEquals0(Object e1, Object e2) { + if (e1.getClass().isArray() && e2.getClass().isArray()) { + return deepEqualsArray(e1, e2); + } else if (e1 instanceof Iterable && e2 instanceof Iterable) { + return deepEqualsIterable((Iterable) e1, (Iterable) e2); + } else if (e1 instanceof Tuple && e2 instanceof Tuple) { + return deepEqualsTuple((Tuple) e1, (Tuple) e2); + } else if (e1 instanceof Row && e2 instanceof Row) { + return deepEqualsRow((Row) e1, (Row) e2); + } else if (e1 instanceof Throwable && e2 instanceof Throwable) { + return ((Throwable) e1).getMessage().equals(((Throwable) e2).getMessage()); + } else { + return e1.equals(e2); + } + } + + private boolean deepEqualsIterable(Iterable i1, Iterable i2) { + Iterator s1 = i1.iterator(); + Iterator s2 = i2.iterator(); + + while (s1.hasNext() && s2.hasNext()) { + Object l = s1.next(); + Object r = s2.next(); + + if (!deepEquals(l, r)) { + return false; + } + } + return !s1.hasNext() && !s2.hasNext(); + } + + private boolean deepEqualsTuple(Tuple tuple1, Tuple tuple2) { + if (tuple1.getArity() != tuple2.getArity()) { + return false; + } + + for (int i = 0; i < tuple1.getArity(); i++) { + Object o1 = tuple1.getField(i); + Object o2 = tuple2.getField(i); + + if (!deepEquals(o1, o2)) { + return false; + } + } + + return true; + } + + private boolean deepEqualsArray(Object array1, Object array2) { + int length1 = Array.getLength(array1); + int length2 = Array.getLength(array2); + + if (length1 != length2) { + return false; + } + + for (int i = 0; i < length1; i++) { + Object o1 = Array.get(array1, i); + Object o2 = Array.get(array2, i); + + if (!deepEquals(o1, o2)) { + return false; + } + } + + return true; + } + + private boolean deepEqualsRow(Row row1, Row row2) { + int arity = row1.getArity(); + + if (row1.getArity() != row2.getArity()) { + return false; + } + + for (int i = 0; i < arity; i++) { + Object copiedValue = row1.getField(i); + Object element = row2.getField(i); + if (!deepEquals(copiedValue, element)) { + return false; + } + } + + return true; + } +} diff --git a/flink-core/src/test/java/org/apache/flink/testutils/EntropyInjectingTestFileSystem.java b/flink-core/src/test/java/org/apache/flink/testutils/EntropyInjectingTestFileSystem.java new file mode 100644 index 0000000000000..ef1575bea7446 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/testutils/EntropyInjectingTestFileSystem.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.testutils; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.EntropyInjectingFileSystem; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.FileSystemFactory; +import org.apache.flink.core.fs.local.LocalFileSystem; + +import java.net.URI; + +/** + * A test file system that implements {@link EntropyInjectingFileSystem}. + */ +public class EntropyInjectingTestFileSystem extends LocalFileSystem implements EntropyInjectingFileSystem { + + public static final String ENTROPY_INJECTION_KEY = "_entropy_"; + + public static final String ENTROPY = "_resolved_"; + + @Override + public String getEntropyInjectionKey() { + return ENTROPY_INJECTION_KEY; + } + + @Override + public String generateEntropy() { + return ENTROPY; + } + + public static class EntropyInjectingTestFileSystemFactory implements FileSystemFactory { + + @Override + public String getScheme() { + return "test-entropy"; + } + + @Override + public void configure(final Configuration config) { + } + + @Override + public FileSystem create(final URI fsUri) { + return new EntropyInjectingTestFileSystem(); + } + } +} diff --git a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java index 0e8639109c754..ccbb5d7d9ab4f 100644 --- a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java +++ b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java @@ -23,15 +23,21 @@ import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.testutils.ClassLoaderUtils; import org.apache.flink.types.DoubleValue; import org.apache.flink.types.StringValue; import org.apache.flink.types.Value; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Proxy; +import java.net.URLClassLoader; import java.util.Objects; import java.util.Random; @@ -46,6 +52,50 @@ */ public class InstantiationUtilTest extends TestLogger { + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final String PROXY_DEFINITION_FORMAT = + "import java.lang.reflect.InvocationHandler;" + + "import java.lang.reflect.Method;" + + "import java.io.Serializable;" + + "public class %s implements InvocationHandler, Serializable {\n" + + "\n" + + " @Override\n" + + " public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {\n" + + " return null;\n" + + " }\n" + + "}"; + + @Test + public void testResolveProxyClass() throws Exception { + final String interfaceName = "UserDefinedInterface"; + final String proxyName = "UserProxy"; + + try (URLClassLoader userClassLoader = createClassLoader(interfaceName, proxyName)) { + Class userInterface = Class.forName(interfaceName, false, userClassLoader); + InvocationHandler userProxy = (InvocationHandler) Class.forName(proxyName, false, userClassLoader) + .newInstance(); + + Object proxy = Proxy.newProxyInstance(userClassLoader, new Class[]{userInterface}, userProxy); + + byte[] serializeObject = InstantiationUtil.serializeObject(proxy); + Object deserializedProxy = InstantiationUtil.deserializeObject(serializeObject, userClassLoader); + assertNotNull(deserializedProxy); + } + } + + private URLClassLoader createClassLoader(String interfaceName, String proxyName) throws IOException { + return ClassLoaderUtils.withRoot(temporaryFolder.newFolder()) + .addClass(interfaceName, String.format("interface %s { void test();}", interfaceName)) + .addClass(proxyName, createProxyDefinition(proxyName)) + .build(); + } + + private String createProxyDefinition(String proxyName) { + return String.format(PROXY_DEFINITION_FORMAT, proxyName); + } + @Test public void testInstantiationOfStringValue() { StringValue stringValue = InstantiationUtil.instantiate( diff --git a/flink-core/src/test/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory b/flink-core/src/test/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory index 5a3a31de23566..76965cb214543 100644 --- a/flink-core/src/test/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory +++ b/flink-core/src/test/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.flink.testutils.TestFileSystem$TestFileSystemFactory \ No newline at end of file +org.apache.flink.testutils.TestFileSystem$TestFileSystemFactory +org.apache.flink.testutils.EntropyInjectingTestFileSystem$EntropyInjectingTestFileSystemFactory diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index c11517af53f93..deedb3027a900 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -106,7 +106,7 @@ under the License. org.apache.flink - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} ${project.version} @@ -207,6 +207,20 @@ under the License. provided + + org.apache.flink + flink-examples-streaming-state-machine_${scala.binary.version} + ${project.version} + provided + + + + org.apache.flink + flink-examples-streaming-twitter_${scala.binary.version} + ${project.version} + provided + + org.apache.flink flink-gelly-examples_${scala.binary.version} @@ -228,13 +242,6 @@ under the License. provided - - org.apache.flink - flink-metrics-ganglia - ${project.version} - provided - - org.apache.flink flink-metrics-graphite @@ -244,7 +251,7 @@ under the License. org.apache.flink - flink-metrics-prometheus + flink-metrics-prometheus_${scala.binary.version} ${project.version} provided @@ -542,7 +549,9 @@ under the License. - + + Apache Flink + diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml index 89228b39271f8..438321c9ef4cb 100644 --- a/flink-dist/src/main/assemblies/bin.xml +++ b/flink-dist/src/main/assemblies/bin.xml @@ -58,9 +58,27 @@ under the License. conf 0644 + + + ../LICENSE + + 0644 + + + + ../NOTICE-binary + + NOTICE + 0644 + + + ../licenses-binary + licenses + 0644 + src/main/flink-bin/bin @@ -130,17 +148,6 @@ under the License. - - - ../ - - 0644 - - LICENSE* - NOTICE* - - - @@ -187,6 +194,30 @@ under the License. + + + ../flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/target + examples/streaming + 0644 + + *.jar + + + original-*.jar + + + + ../flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/target + examples/streaming + 0644 + + *.jar + + + original-*.jar + + + ../flink-libraries/flink-gelly-examples/target diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml index 6fe661a6c46da..0d9acf3c3ae0e 100644 --- a/flink-dist/src/main/assemblies/opt.xml +++ b/flink-dist/src/main/assemblies/opt.xml @@ -85,28 +85,14 @@ - ../flink-metrics/flink-metrics-dropwizard/target/flink-metrics-dropwizard-${project.version}-jar-with-dependencies.jar - opt/ - flink-metrics-dropwizard-${project.version}.jar - 0644 - - - - ../flink-metrics/flink-metrics-ganglia/target/flink-metrics-ganglia-${project.version}-jar-with-dependencies.jar - opt/ - flink-metrics-ganglia-${project.version}.jar - 0644 - - - - ../flink-metrics/flink-metrics-graphite/target/flink-metrics-graphite-${project.version}-jar-with-dependencies.jar + ../flink-metrics/flink-metrics-graphite/target/flink-metrics-graphite-${project.version}.jar opt/ flink-metrics-graphite-${project.version}.jar 0644 - ../flink-metrics/flink-metrics-prometheus/target/flink-metrics-prometheus-${project.version}.jar + ../flink-metrics/flink-metrics-prometheus/target/flink-metrics-prometheus_${scala.binary.version}-${project.version}.jar opt/ flink-metrics-prometheus-${project.version}.jar 0644 diff --git a/flink-dist/src/main/resources/META-INF/NOTICE b/flink-dist/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..442787a8b516d --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/NOTICE @@ -0,0 +1,60 @@ +flink-dist +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.code.findbugs:jsr305:1.3.9 +- com.twitter:chill-java:0.7.6 +- com.twitter:chill_2.11:0.7.6 +- com.typesafe:config:1.3.0 +- com.typesafe:ssl-config-core_2.11:0.2.1 +- com.typesafe.akka:akka-actor_2.11:2.4.20 +- com.typesafe.akka:akka-camel_2.11:2.4.20 +- com.typesafe.akka:akka-protobuf_2.11:2.4.20 +- com.typesafe.akka:akka-slf4j_2.11:2.4.20 +- com.typesafe.akka:akka-stream_2.11:2.4.20 +- commons-cli:commons-cli:1.3.1 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- org.apache.camel:camel-core:2.17.7 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-lang3:3.3.2 +- org.apache.commons:commons-math3:3.5 +- org.javassist:javassist:3.19.0-GA +- org.objenesis:objenesis:2.1 +- org.rocksdb:rocksdbjni:5.7.5 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.esotericsoftware.kryo:kryo:2.24.0 +- com.esotericsoftware.minlog:minlog:1.2 +- org.clapper:grizzled-slf4j_2.11:1.3.2 + +The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala. + +- org.scala-lang:scala-compiler:2.11.12 +- org.scala-lang:scala-library:2.11.12 +- org.scala-lang:scala-reflect:2.11.12 +- org.scala-lang.modules:scala-java8-compat_2.11:0.7.0 +- org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4 +- org.scala-lang.modules:scala-xml_2.11:1.0.5 + +This project bundles the following dependencies under the MIT/X11 license. +See bundled license files for details. + +- com.github.scopt:scopt_2.11:3.5.0 +- org.slf4j:slf4j-api:1.7.15 + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles the following dependencies under the Creative Commons CC0 "No Rights Reserved". + +- org.reactivestreams:reactive-streams:1.0.0 diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.grizzled-slf4j b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.grizzled-slf4j new file mode 100644 index 0000000000000..6601c0131eeac --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.grizzled-slf4j @@ -0,0 +1,11 @@ +Copyright © 2010-2012 Brian M. Clapper. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +Neither the name “clapper.org” nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.kryo b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.kryo new file mode 100644 index 0000000000000..e1cd88478edf7 --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.kryo @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.minlog b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.minlog new file mode 100644 index 0000000000000..e1cd88478edf7 --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.minlog @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scala b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scala new file mode 100644 index 0000000000000..6d8bdabbbe0fb --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scala @@ -0,0 +1,11 @@ +Copyright (c) 2002- EPFL +Copyright (c) 2011- Lightbend, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +Neither the name of the EPFL nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scopt b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scopt new file mode 100644 index 0000000000000..6d6a875878f4b --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.scopt @@ -0,0 +1,21 @@ +This project is licensed under the [MIT license](https://en.wikipedia.org/wiki/MIT_License). + +Copyright (c) scopt contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-dist/src/main/resources/META-INF/licenses/LICENSE.slf4j-api b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.slf4j-api new file mode 100644 index 0000000000000..93119e70ed5e7 --- /dev/null +++ b/flink-dist/src/main/resources/META-INF/licenses/LICENSE.slf4j-api @@ -0,0 +1,21 @@ +Copyright (c) 2004-2017 QOS.ch + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-dist/src/main/resources/flink-conf.yaml b/flink-dist/src/main/resources/flink-conf.yaml index efdf33e3852a9..99ea6851165f3 100644 --- a/flink-dist/src/main/resources/flink-conf.yaml +++ b/flink-dist/src/main/resources/flink-conf.yaml @@ -121,17 +121,20 @@ parallelism.default: 1 # state.backend.incremental: false #============================================================================== -# Web Frontend +# Rest & web frontend #============================================================================== -# The address under which the web-based runtime monitor listens. +# The port to which the REST client connects to and the server binds to. # -#web.address: 0.0.0.0 +#rest.port: 8081 -# The port under which the web-based runtime monitor listens. -# A value of -1 deactivates the web server. +# The address to which the REST client will connect to +# +#rest.address: 0.0.0.0 -rest.port: 8081 +# The address that the REST & web server binds to +# +#rest.bind-address: 0.0.0.0 # Flag to specify whether job submission is enabled from the web-based # runtime monitor. Uncomment to disable. diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml index 75e3f9b53fb98..5c03fff2301bc 100644 --- a/flink-docs/pom.xml +++ b/flink-docs/pom.xml @@ -55,7 +55,7 @@ under the License. org.apache.flink - flink-metrics-prometheus + flink-metrics-prometheus_${scala.binary.version} ${project.version} diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java index d0f665a5051e4..ed8c2378ac0b3 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java @@ -202,7 +202,7 @@ static List extractConfigOptions(Class clazz) { List configOptions = new ArrayList<>(8); Field[] fields = clazz.getFields(); for (Field field : fields) { - if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) { + if (isConfigOption(field) && shouldBeDocumented(field)) { configOptions.add(new OptionWithMetaInfo((ConfigOption) field.get(null), field)); } } @@ -213,6 +213,15 @@ static List extractConfigOptions(Class clazz) { } } + private static boolean isConfigOption(Field field) { + return field.getType().equals(ConfigOption.class); + } + + private static boolean shouldBeDocumented(Field field) { + return field.getAnnotation(Deprecated.class) == null && + field.getAnnotation(Documentation.ExcludeFromDocumentation.class) == null; + } + /** * Transforms this configuration group into HTML formatted table. * Options are sorted alphabetically by key. @@ -323,11 +332,20 @@ List getDefaultOptions() { private Node findGroupRoot(String key) { String[] keyComponents = key.split("\\."); + Node lastRootNode = root; Node currentNode = root; for (String keyComponent : keyComponents) { - currentNode = currentNode.findChild(keyComponent); + final Node childNode = currentNode.getChild(keyComponent); + if (childNode == null) { + break; + } else { + currentNode = childNode; + if (currentNode.isGroupRoot()) { + lastRootNode = currentNode; + } + } } - return currentNode.isGroupRoot() ? currentNode : root; + return lastRootNode; } private static class Node { @@ -344,12 +362,8 @@ private Node addChild(String keyComponent) { return child; } - private Node findChild(String keyComponent) { - Node child = children.get(keyComponent); - if (child == null) { - return this; - } - return child; + private Node getChild(String keyComponent) { + return children.get(keyComponent); } private void assignOption(OptionWithMetaInfo option) { diff --git a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java index a0eade69d455f..438c10848b2b5 100644 --- a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java +++ b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java @@ -38,7 +38,10 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.List; +import java.util.Map; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; /** @@ -90,6 +93,56 @@ public void testCreatingDescription() { assertEquals(expectedTable, htmlTable); } + @ConfigGroups(groups = { + @ConfigGroup(name = "group1", keyPrefix = "a.b"), + @ConfigGroup(name = "group2", keyPrefix = "a.b.c.d")}) + static class TestConfigPrefix { + // should end up in the default group + public static ConfigOption option1 = ConfigOptions + .key("a.option") + .defaultValue(2); + + // should end up in group1, perfect key-prefix match + public static ConfigOption option2 = ConfigOptions + .key("a.b.option") + .noDefaultValue(); + + // should end up in group1, full key-prefix match + public static ConfigOption option3 = ConfigOptions + .key("a.b.c.option") + .defaultValue(2); + + // should end up in group1, full key-prefix match for group 1, partial match for group 2 + // checks that the generator remembers the last encountered root node + public static ConfigOption option4 = ConfigOptions + .key("a.b.c.e.option") + .defaultValue(2); + + // should end up in the default group, since no group exists with prefix "a.c" + // checks that the generator does not ignore components (like ignoring "c" to find a match "a.b") + public static ConfigOption option5 = ConfigOptions + .key("a.c.b.option") + .noDefaultValue(); + } + + @Test + public void testLongestPrefixMatching() { + final List> tables = ConfigOptionsDocGenerator.generateTablesForClass( + TestConfigPrefix.class); + + assertEquals(3, tables.size()); + final Map tablesConverted = new HashMap<>(tables.size()); + for (final Tuple2 table : tables) { + tablesConverted.put(table.f0 != null ? table.f0.name() : "default", table.f1); + } + + assertThat(tablesConverted.get("group1"), containsString("a.b.option")); + assertThat(tablesConverted.get("group1"), containsString("a.b.c.option")); + assertThat(tablesConverted.get("group1"), containsString("a.b.c.e.option")); + assertThat(tablesConverted.get("default"), containsString("a.option")); + assertThat(tablesConverted.get("default"), containsString("a.c.b.option")); + } + @ConfigGroups(groups = { @ConfigGroup(name = "firstGroup", keyPrefix = "first"), @ConfigGroup(name = "secondGroup", keyPrefix = "second")}) @@ -266,4 +319,45 @@ public void testCommonOptions() throws IOException, ClassNotFoundException { assertEquals(expected, output); } + + static class TestConfigGroupWithExclusion { + public static ConfigOption firstOption = ConfigOptions + .key("first.option.a") + .defaultValue(2) + .withDescription("This is example description for the first option."); + + @Documentation.ExcludeFromDocumentation + public static ConfigOption excludedOption = ConfigOptions + .key("excluded.option.a") + .noDefaultValue() + .withDescription("This should not be documented."); + } + + /** + * Tests that {@link ConfigOption} annotated with {@link Documentation.ExcludeFromDocumentation} + * are not documented. + */ + @Test + public void testConfigOptionExclusion() { + final String expectedTable = + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
KeyDefaultDescription
first.option.a
2This is example description for the first option.
\n"; + final String htmlTable = ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroupWithExclusion.class).get(0).f1; + + assertEquals(expectedTable, htmlTable); + } } diff --git a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocsCompletenessITCase.java b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocsCompletenessITCase.java index 25ce3dca4d1b3..bfc55c1ebbdd1 100644 --- a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocsCompletenessITCase.java +++ b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocsCompletenessITCase.java @@ -171,6 +171,7 @@ private static Map parseDocumentedOptions() throws IOE private static Collection parseDocumentedOptionsFromFile(Path file) throws IOException { Document document = Jsoup.parse(file.toFile(), StandardCharsets.UTF_8.name()); + document.outputSettings().syntax(Document.OutputSettings.Syntax.xml); document.outputSettings().prettyPrint(false); return document.getElementsByTag("table").stream() .map(element -> element.getElementsByTag("tbody").get(0)) diff --git a/flink-end-to-end-tests/README.md b/flink-end-to-end-tests/README.md index 257298e3cceec..0afc96f227654 100644 --- a/flink-end-to-end-tests/README.md +++ b/flink-end-to-end-tests/README.md @@ -47,6 +47,8 @@ In order to add a new test case you need add it to either `test-scripts/run-nigh _Note: If you want to parameterize your tests please do so by adding multiple test cases with parameters as arguments to the nightly / pre-commit test suites. This allows the test runner to do a cleanup in between each individual test and also to fail those tests individually._ +_Note: While developing a new test case make sure to enable bash's error handling in `test-scripts/common.sh` by uncommenting `set -Eexuo pipefail` and commenting the current default `set` call. Once your test is implemented properly, add `set -Eeuo pipefail` on the very top of your test script (before any `common` script)._ + ### Passing your test A test is considered to have passed if it: - has exit code 0 diff --git a/flink-end-to-end-tests/flink-cli-test/pom.xml b/flink-end-to-end-tests/flink-cli-test/pom.xml new file mode 100644 index 0000000000000..ebb42322e87aa --- /dev/null +++ b/flink-end-to-end-tests/flink-cli-test/pom.xml @@ -0,0 +1,73 @@ + + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + + 4.0.0 + + flink-cli-test_${scala.binary.version} + flink-cli-test + jar + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + provided + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + PeriodicStreamingJob + package + + shade + + + PeriodicStreamingJob + + + org.apache.flink.streaming.tests.PeriodicStreamingJob + + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-cli-test/src/main/java/org/apache/flink/streaming/tests/PeriodicStreamingJob.java b/flink-end-to-end-tests/flink-cli-test/src/main/java/org/apache/flink/streaming/tests/PeriodicStreamingJob.java new file mode 100644 index 0000000000000..7cabf849aaf03 --- /dev/null +++ b/flink-end-to-end-tests/flink-cli-test/src/main/java/org/apache/flink/streaming/tests/PeriodicStreamingJob.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.checkpoint.ListCheckpointed; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.windowing.time.Time; + +import java.util.Collections; +import java.util.List; + +/** + * This is a periodic streaming job that runs for CLI testing purposes. + * + *

The stream is bounded and will complete after the specified duration. + * + *

Parameters: + * -outputPath Sets the path to where the result data is written. + * -recordsPerSecond Sets the output record frequency. + * -durationInSecond Sets the running duration of the job. + * -offsetInSecond Sets the startup delay before processing the first message. + */ +public class PeriodicStreamingJob { + + public static void main(String[] args) throws Exception { + ParameterTool params = ParameterTool.fromArgs(args); + String outputPath = params.getRequired("outputPath"); + int recordsPerSecond = params.getInt("recordsPerSecond", 10); + int duration = params.getInt("durationInSecond", 60); + int offset = params.getInt("offsetInSecond", 0); + + StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + sEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); + sEnv.enableCheckpointing(4000); + sEnv.getConfig().setAutoWatermarkInterval(1000); + + // execute a simple pass through program. + PeriodicSourceGenerator generator = new PeriodicSourceGenerator( + recordsPerSecond, duration, offset); + DataStream rows = sEnv.addSource(generator); + + DataStream result = rows + .keyBy(1) + .timeWindow(Time.seconds(5)) + .sum(0); + + result.writeAsText(outputPath + "/result.txt", FileSystem.WriteMode.OVERWRITE) + .setParallelism(1); + + sEnv.execute(); + } + + /** + * Data-generating source function. + */ + public static class PeriodicSourceGenerator implements SourceFunction, ResultTypeQueryable, ListCheckpointed { + private final int sleepMs; + private final int durationMs; + private final int offsetSeconds; + private long ms = 0; + + public PeriodicSourceGenerator(float rowsPerSecond, int durationSeconds, int offsetSeconds) { + this.durationMs = durationSeconds * 1000; + this.sleepMs = (int) (1000 / rowsPerSecond); + this.offsetSeconds = offsetSeconds; + } + + @Override + public void run(SourceContext ctx) throws Exception { + long offsetMs = offsetSeconds * 1000L; + + while (ms < durationMs) { + synchronized (ctx.getCheckpointLock()) { + ctx.collect(new Tuple2<>(ms + offsetMs, "key")); + } + ms += sleepMs; + Thread.sleep(sleepMs); + } + } + + @Override + public void cancel() { + // nothing to do + } + + @Override + public TypeInformation getProducedType() { + return Types.TUPLE(Types.LONG, Types.STRING); + } + + @Override + public List snapshotState(long checkpointId, long timestamp) { + return Collections.singletonList(ms); + } + + @Override + public void restoreState(List state) { + for (Long l : state) { + ms += l; + } + } + } +} diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java index fb92960bb8684..af39522aff96d 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java @@ -24,20 +24,27 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.KeyedStream; import org.apache.flink.streaming.api.datastream.WindowedStream; import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; +import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.streaming.tests.artificialstate.ArtificalOperatorStateMapper; @@ -49,6 +56,10 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.flink.streaming.tests.TestOperatorEnum.EVENT_IDENTITY_MAPPER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.MAPPER_RETURNS_OUT_WITH_CUSTOM_SER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.RESULT_TYPE_QUERYABLE_MAPPER_WITH_CUSTOM_SER; + /** * A factory for components of general purpose test jobs for Flink's DataStream API operators and primitives. * @@ -86,9 +97,13 @@ *

  • sequence_generator_source.event_time.clock_progress (long, default - 100): The amount of event time to progress per event generated by the sequence generator.
    • *
  • tumbling_window_operator.num_events (long, default - 20L): The duration of the window, indirectly determined by the target number of events in each window. * Total duration is (sliding_window_operator.num_events) * (sequence_generator_source.event_time.clock_progress).
    • + *
  • test_slide_factor (int, default - 3): test_slide_factor (int, default - 3): how many slides there are in a + * single window (in other words at most how many windows may be opened at time for a given key) The length of + * a window will be calculated as (test_slide_size) * (test_slide_factor)
    • + *
  • test_slide_size (long, default - 250): length of a slide of sliding window in milliseconds. The length of a window will be calculated as (test_slide_size) * (test_slide_factor)
    • * */ -class DataStreamAllroundTestJobFactory { +public class DataStreamAllroundTestJobFactory { private static final ConfigOption TEST_SEMANTICS = ConfigOptions .key("test.semantics") .defaultValue("exactly-once") @@ -201,7 +216,15 @@ class DataStreamAllroundTestJobFactory { .key("tumbling_window_operator.num_events") .defaultValue(20L); - static void setupEnvironment(StreamExecutionEnvironment env, ParameterTool pt) throws Exception { + private static final ConfigOption TEST_SLIDE_FACTOR = ConfigOptions + .key("test_slide_factor") + .defaultValue(3); + + private static final ConfigOption TEST_SLIDE_SIZE = ConfigOptions + .key("test_slide_size") + .defaultValue(250L); + + public static void setupEnvironment(StreamExecutionEnvironment env, ParameterTool pt) throws Exception { // set checkpointing semantics String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue()); @@ -254,13 +277,13 @@ static void setupEnvironment(StreamExecutionEnvironment env, ParameterTool pt) t STATE_BACKEND_FILE_ASYNC.key(), STATE_BACKEND_FILE_ASYNC.defaultValue()); - env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints)); + env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints)); } else if ("rocks".equalsIgnoreCase(stateBackend)) { boolean incrementalCheckpoints = pt.getBoolean( STATE_BACKEND_ROCKS_INCREMENTAL.key(), STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue()); - env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints)); + env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints)); } else { throw new IllegalArgumentException("Unknown backend requested: " + stateBackend); } @@ -322,6 +345,8 @@ static BoundedOutOfOrdernessTimestampExtractor createTimestampExtractor(P SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.key(), SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.defaultValue()))) { + private static final long serialVersionUID = -3154419724891779938L; + @Override public long extractTimestamp(Event element) { return element.getEventTime(); @@ -367,8 +392,8 @@ static boolean isSimulateFailures(ParameterTool pt) { return pt.getBoolean(TEST_SIMULATE_FAILURE.key(), TEST_SIMULATE_FAILURE.defaultValue()); } - static MapFunction createExceptionThrowingFailureMapper(ParameterTool pt) { - return new ExceptionThrowingFailureMapper<>( + static MapFunction createFailureMapper(ParameterTool pt) { + return new FailureMapper<>( pt.getLong( TEST_SIMULATE_FAILURE_NUM_RECORDS.key(), TEST_SIMULATE_FAILURE_NUM_RECORDS.defaultValue()), @@ -425,7 +450,7 @@ static ArtificalOperatorStateMapper createArtificialOperatorS return new ArtificalOperatorStateMapper<>(mapFunction); } - static ArtificialStateBuilder createValueStateBuilder( + private static ArtificialStateBuilder createValueStateBuilder( JoinFunction inputAndOldStateToNewState, ValueStateDescriptor valueStateDescriptor) { @@ -435,7 +460,7 @@ static ArtificialStateBuilder createValueStateBuilder( valueStateDescriptor); } - static ArtificialStateBuilder createListStateBuilder( + private static ArtificialStateBuilder createListStateBuilder( JoinFunction inputAndOldStateToNewState, ListStateDescriptor listStateDescriptor) { @@ -453,4 +478,76 @@ static ArtificialStateBuilder createListStateBuilder( listStateGenerator, listStateDescriptor); } + + static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) { + long slideSize = pt.getLong( + TEST_SLIDE_SIZE.key(), + TEST_SLIDE_SIZE.defaultValue()); + + long slideFactor = pt.getInt( + TEST_SLIDE_FACTOR.key(), + TEST_SLIDE_FACTOR.defaultValue() + ); + + return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize)); + } + + static FlatMapFunction>, String> createSlidingWindowCheckMapper(ParameterTool pt) { + return new SlidingWindowCheckMapper(pt.getInt( + TEST_SLIDE_FACTOR.key(), + TEST_SLIDE_FACTOR.defaultValue() + )); + } + + static DataStream verifyCustomStatefulTypeSerializer(DataStream eventStream) { + return eventStream + .map(new EventIdentityFunctionWithCustomEventTypeInformation()) + .name(RESULT_TYPE_QUERYABLE_MAPPER_WITH_CUSTOM_SER.getName()) + .uid(RESULT_TYPE_QUERYABLE_MAPPER_WITH_CUSTOM_SER.getUid()) + // apply a keyBy so that we have a non-chained operator with Event as input type that goes through serialization + .keyBy(new EventKeySelectorWithCustomKeyTypeInformation()) + + .map(e -> e) + .returns(new SingleThreadAccessCheckingTypeInfo<>(Event.class)) + .name(MAPPER_RETURNS_OUT_WITH_CUSTOM_SER.getName()) + .uid(MAPPER_RETURNS_OUT_WITH_CUSTOM_SER.getUid()) + // apply a keyBy so that we have a non-chained operator with Event as input type that goes through serialization + .keyBy(new EventKeySelectorWithCustomKeyTypeInformation()) + + .map(e -> e) + .name(EVENT_IDENTITY_MAPPER.getName()) + .uid(EVENT_IDENTITY_MAPPER.getUid()); + } + + private static class EventIdentityFunctionWithCustomEventTypeInformation + implements MapFunction, ResultTypeQueryable { + + private final SingleThreadAccessCheckingTypeInfo typeInformation = new SingleThreadAccessCheckingTypeInfo<>(Event.class); + + @Override + public Event map(Event value) { + return value; + } + + @Override + public TypeInformation getProducedType() { + return typeInformation; + } + } + + private static class EventKeySelectorWithCustomKeyTypeInformation + implements KeySelector, ResultTypeQueryable { + + private final SingleThreadAccessCheckingTypeInfo typeInformation = new SingleThreadAccessCheckingTypeInfo<>(Integer.class); + + @Override + public Integer getKey(Event value) { + return value.getKey(); + } + + @Override + public TypeInformation getProducedType() { + return typeInformation; + } + } } diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestProgram.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestProgram.java index 70fdade4a1c67..3138af5de7363 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestProgram.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestProgram.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.formats.avro.typeutils.AvroSerializer; @@ -36,16 +37,32 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.applyTumblingWindows; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createArtificialKeyedStateMapper; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createArtificialOperatorStateMapper; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createEventSource; -import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createExceptionThrowingFailureMapper; +import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createFailureMapper; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createSemanticsCheckMapper; +import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createSlidingWindow; +import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createSlidingWindowCheckMapper; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.createTimestampExtractor; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.isSimulateFailures; import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.setupEnvironment; +import static org.apache.flink.streaming.tests.TestOperatorEnum.EVENT_SOURCE; +import static org.apache.flink.streaming.tests.TestOperatorEnum.FAILURE_MAPPER_NAME; +import static org.apache.flink.streaming.tests.TestOperatorEnum.KEYED_STATE_OPER_WITH_AVRO_SER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.OPERATOR_STATE_OPER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.SEMANTICS_CHECK_MAPPER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.SEMANTICS_CHECK_PRINT_SINK; +import static org.apache.flink.streaming.tests.TestOperatorEnum.SLIDING_WINDOW_AGG; +import static org.apache.flink.streaming.tests.TestOperatorEnum.SLIDING_WINDOW_CHECK_MAPPER; +import static org.apache.flink.streaming.tests.TestOperatorEnum.SLIDING_WINDOW_CHECK_PRINT_SINK; +import static org.apache.flink.streaming.tests.TestOperatorEnum.TIME_WINDOW_OPER; /** * A general purpose test job for Flink's DataStream API operators and primitives. @@ -64,11 +81,6 @@ * */ public class DataStreamAllroundTestProgram { - private static final String KEYED_STATE_OPER_NAME = "ArtificalKeyedStateMapper"; - private static final String OPERATOR_STATE_OPER_NAME = "ArtificalOperatorStateMapper"; - private static final String TIME_WINDOW_OPER_NAME = "TumblingWindowOperator"; - private static final String SEMANTICS_CHECK_MAPPER_NAME = "SemanticsCheckMapper"; - private static final String FAILURE_MAPPER_NAME = "ExceptionThrowingFailureMapper"; public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); @@ -78,7 +90,10 @@ public static void main(String[] args) throws Exception { setupEnvironment(env, pt); // add a keyed stateful map operator, which uses Kryo for state serialization - DataStream eventStream = env.addSource(createEventSource(pt)) + DataStream eventStream = env + .addSource(createEventSource(pt)) + .name(EVENT_SOURCE.getName()) + .uid(EVENT_SOURCE.getUid()) .assignTimestampsAndWatermarks(createTimestampExtractor(pt)) .keyBy(Event::getKey) .map(createArtificialKeyedStateMapper( @@ -86,18 +101,21 @@ public static void main(String[] args) throws Exception { (MapFunction) in -> in, // state is verified and updated per event as a wrapped ComplexPayload state object (Event event, ComplexPayload lastState) -> { - if (lastState != null && !lastState.getStrPayload().equals(KEYED_STATE_OPER_NAME) + if (lastState != null && !lastState.getStrPayload().equals(KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER.getName()) && lastState.getInnerPayLoad().getSequenceNumber() == (event.getSequenceNumber() - 1)) { System.out.println("State is set or restored incorrectly"); } - return new ComplexPayload(event, KEYED_STATE_OPER_NAME); + return new ComplexPayload(event, KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER.getName()); }, Arrays.asList( new KryoSerializer<>(ComplexPayload.class, env.getConfig()), // KryoSerializer new StatefulComplexPayloadSerializer()), // custom stateful serializer Collections.singletonList(ComplexPayload.class) // KryoSerializer via type extraction ) - ).returns(Event.class).name(KEYED_STATE_OPER_NAME + "_Kryo_and_Custom_Stateful"); + ) + .returns(Event.class) + .name(KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER.getName()) + .uid(KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER.getUid()); // add a keyed stateful map operator, which uses Avro for state serialization eventStream = eventStream @@ -107,7 +125,7 @@ public static void main(String[] args) throws Exception { (MapFunction) in -> in, // state is verified and updated per event as a wrapped ComplexPayloadAvro state object (Event event, ComplexPayloadAvro lastState) -> { - if (lastState != null && !lastState.getStrPayload().equals(KEYED_STATE_OPER_NAME) + if (lastState != null && !lastState.getStrPayload().equals(KEYED_STATE_OPER_WITH_AVRO_SER.getName()) && lastState.getInnerPayLoad().getSequenceNumber() == (event.getSequenceNumber() - 1)) { System.out.println("State is set or restored incorrectly"); } @@ -115,7 +133,7 @@ public static void main(String[] args) throws Exception { ComplexPayloadAvro payload = new ComplexPayloadAvro(); payload.setEventTime(event.getEventTime()); payload.setInnerPayLoad(new InnerPayLoadAvro(event.getSequenceNumber())); - payload.setStrPayload(KEYED_STATE_OPER_NAME); + payload.setStrPayload(KEYED_STATE_OPER_WITH_AVRO_SER.getName()); payload.setStringList(Arrays.asList(String.valueOf(event.getKey()), event.getPayload())); return payload; @@ -124,36 +142,75 @@ public static void main(String[] args) throws Exception { new AvroSerializer<>(ComplexPayloadAvro.class)), // custom AvroSerializer Collections.singletonList(ComplexPayloadAvro.class) // AvroSerializer via type extraction ) - ).returns(Event.class).name(KEYED_STATE_OPER_NAME + "_Avro"); + ) + .returns(Event.class) + .name(KEYED_STATE_OPER_WITH_AVRO_SER.getName()) + .uid(KEYED_STATE_OPER_WITH_AVRO_SER.getUid()); DataStream eventStream2 = eventStream .map(createArtificialOperatorStateMapper((MapFunction) in -> in)) - .name(OPERATOR_STATE_OPER_NAME) - .returns(Event.class); + .returns(Event.class) + .name(OPERATOR_STATE_OPER.getName()) + .uid(OPERATOR_STATE_OPER.getUid()); // apply a tumbling window that simply passes forward window elements; // this allows the job to cover timers state + @SuppressWarnings("Convert2Lambda") DataStream eventStream3 = applyTumblingWindows(eventStream2.keyBy(Event::getKey), pt) .apply(new WindowFunction() { @Override - public void apply(Integer integer, TimeWindow window, Iterable input, Collector out) throws Exception { + public void apply(Integer integer, TimeWindow window, Iterable input, Collector out) { for (Event e : input) { out.collect(e); } } - }).name(TIME_WINDOW_OPER_NAME); + }) + .name(TIME_WINDOW_OPER.getName()) + .uid(TIME_WINDOW_OPER.getUid()); + + eventStream3 = DataStreamAllroundTestJobFactory.verifyCustomStatefulTypeSerializer(eventStream3); if (isSimulateFailures(pt)) { eventStream3 = eventStream3 - .map(createExceptionThrowingFailureMapper(pt)) + .map(createFailureMapper(pt)) .setParallelism(1) - .name(FAILURE_MAPPER_NAME); + .name(FAILURE_MAPPER_NAME.getName()) + .uid(FAILURE_MAPPER_NAME.getUid()); } eventStream3.keyBy(Event::getKey) .flatMap(createSemanticsCheckMapper(pt)) - .name(SEMANTICS_CHECK_MAPPER_NAME) - .addSink(new PrintSinkFunction<>()); + .name(SEMANTICS_CHECK_MAPPER.getName()) + .uid(SEMANTICS_CHECK_MAPPER.getUid()) + .addSink(new PrintSinkFunction<>()) + .name(SEMANTICS_CHECK_PRINT_SINK.getName()) + .uid(SEMANTICS_CHECK_PRINT_SINK.getUid()); + + // Check sliding windows aggregations. Output all elements assigned to a window and later on + // check if each event was emitted slide_factor number of times + DataStream>> eventStream4 = eventStream2.keyBy(Event::getKey) + .window(createSlidingWindow(pt)) + .apply(new WindowFunction>, Integer, TimeWindow>() { + private static final long serialVersionUID = 3166250579972849440L; + + @Override + public void apply( + Integer key, TimeWindow window, Iterable input, + Collector>> out) { + + out.collect(Tuple2.of(key, StreamSupport.stream(input.spliterator(), false).collect(Collectors.toList()))); + } + }) + .name(SLIDING_WINDOW_AGG.getName()) + .uid(SLIDING_WINDOW_AGG.getUid()); + + eventStream4.keyBy(events-> events.f0) + .flatMap(createSlidingWindowCheckMapper(pt)) + .name(SLIDING_WINDOW_CHECK_MAPPER.getName()) + .uid(SLIDING_WINDOW_CHECK_MAPPER.getUid()) + .addSink(new PrintSinkFunction<>()) + .name(SLIDING_WINDOW_CHECK_PRINT_SINK.getName()) + .uid(SLIDING_WINDOW_CHECK_PRINT_SINK.getUid()); env.execute("General purpose test job"); } diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/Event.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/Event.java index 8c219ed5863ce..af7a0ad6a6f0d 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/Event.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/Event.java @@ -18,6 +18,8 @@ package org.apache.flink.streaming.tests; +import java.util.Objects; + /** * The event type of records used in the {@link DataStreamAllroundTestProgram}. */ @@ -51,6 +53,26 @@ public String getPayload() { return payload; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Event event = (Event) o; + return key == event.key && + eventTime == event.eventTime && + sequenceNumber == event.sequenceNumber && + Objects.equals(payload, event.payload); + } + + @Override + public int hashCode() { + return Objects.hash(key, eventTime, sequenceNumber, payload); + } + @Override public String toString() { return "Event{" + diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/ExceptionThrowingFailureMapper.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java similarity index 94% rename from flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/ExceptionThrowingFailureMapper.java rename to flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java index d758ef5cf0d1b..a3a1c253fc070 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/ExceptionThrowingFailureMapper.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java @@ -30,7 +30,7 @@ * of the operator can also be configured. Note that this also takes into account * failures that were not triggered by this mapper, e.g. TaskManager failures. */ -public class ExceptionThrowingFailureMapper extends RichMapFunction implements CheckpointListener { +public class FailureMapper extends RichMapFunction implements CheckpointListener { private static final long serialVersionUID = -5286927943454740016L; @@ -41,7 +41,7 @@ public class ExceptionThrowingFailureMapper extends RichMapFunction imp private long numProcessedRecords; private long numCompleteCheckpoints; - public ExceptionThrowingFailureMapper( + public FailureMapper( long numProcessedRecordsFailureThreshold, long numCompleteCheckpointsFailureThreshold, int maxNumFailures) { diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SemanticsCheckMapper.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SemanticsCheckMapper.java index b67e282334288..1fe5f61ca7359 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SemanticsCheckMapper.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SemanticsCheckMapper.java @@ -34,7 +34,7 @@ public class SemanticsCheckMapper extends RichFlatMapFunction { private static final long serialVersionUID = -744070793650644485L; /** This value state tracks the current sequence number per key. */ - private volatile ValueState sequenceValue; + private transient ValueState sequenceValue; /** This defines how semantics are checked for each update. */ private final ValidatorFunction validator; @@ -60,7 +60,7 @@ public void flatMap(Event event, Collector out) throws Exception { } @Override - public void open(Configuration parameters) throws Exception { + public void open(Configuration parameters) { ValueStateDescriptor sequenceStateDescriptor = new ValueStateDescriptor<>("sequenceState", Long.class); diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SequenceGeneratorSource.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SequenceGeneratorSource.java index c6ecb3f768099..5dd09a9bfc4aa 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SequenceGeneratorSource.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SequenceGeneratorSource.java @@ -28,9 +28,6 @@ import org.apache.flink.util.Preconditions; import org.apache.flink.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -43,8 +40,6 @@ public class SequenceGeneratorSource extends RichParallelSourceFunction i private static final long serialVersionUID = -3986989644799442178L; - private static final Logger LOG = LoggerFactory.getLogger(SequenceGeneratorSource.class); - /** Length of the artificial payload string generated for each event. */ private final int payloadLength; @@ -145,7 +140,7 @@ private void runActive(SourceContext ctx) throws Exception { } } - private void runIdle(SourceContext ctx) throws Exception { + private void runIdle(SourceContext ctx) { ctx.markAsTemporarilyIdle(); // just wait until this source is canceled diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeInfo.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeInfo.java new file mode 100644 index 0000000000000..deb5b0c2893c4 --- /dev/null +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeInfo.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import java.util.Objects; + +/** Custom {@link TypeInformation} to test custom {@link TypeSerializer}. */ +public class SingleThreadAccessCheckingTypeInfo extends TypeInformation { + private final TypeInformation originalTypeInformation; + + SingleThreadAccessCheckingTypeInfo(Class clazz) { + this(TypeInformation.of(clazz)); + } + + private SingleThreadAccessCheckingTypeInfo(TypeInformation originalTypeInformation) { + this.originalTypeInformation = originalTypeInformation; + } + + @Override + public boolean isBasicType() { + return originalTypeInformation.isBasicType(); + } + + @Override + public boolean isTupleType() { + return originalTypeInformation.isTupleType(); + } + + @Override + public int getArity() { + return originalTypeInformation.getArity(); + } + + @Override + public int getTotalFields() { + return originalTypeInformation.getTotalFields(); + } + + @Override + public Class getTypeClass() { + return originalTypeInformation.getTypeClass(); + } + + @Override + public boolean isKeyType() { + return originalTypeInformation.isKeyType(); + } + + @Override + public TypeSerializer createSerializer(ExecutionConfig config) { + return new SingleThreadAccessCheckingTypeSerializer<>(originalTypeInformation.createSerializer(config)); + } + + @Override + public String toString() { + return originalTypeInformation.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o){ + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SingleThreadAccessCheckingTypeInfo that = (SingleThreadAccessCheckingTypeInfo) o; + return Objects.equals(originalTypeInformation, that.originalTypeInformation); + } + + @Override + public int hashCode() { + return Objects.hash(originalTypeInformation); + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof SingleThreadAccessCheckingTypeInfo; + } +} diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeSerializer.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeSerializer.java new file mode 100644 index 0000000000000..b162588dd222e --- /dev/null +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SingleThreadAccessCheckingTypeSerializer.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicReference; + +/** {@link TypeSerializer} to check mutually exclusive thread access API to methods. */ +@Internal +class SingleThreadAccessCheckingTypeSerializer extends TypeSerializer { + private static final long serialVersionUID = 131020282727167064L; + + private final SingleThreadAccessChecker singleThreadAccessChecker; + private final TypeSerializer originalSerializer; + + SingleThreadAccessCheckingTypeSerializer(TypeSerializer originalSerializer) { + this.singleThreadAccessChecker = new SingleThreadAccessChecker(); + this.originalSerializer = originalSerializer; + } + + @Override + public boolean isImmutableType() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.isImmutableType(); + } + } + + @Override + public TypeSerializer duplicate() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return new SingleThreadAccessCheckingTypeSerializer<>(originalSerializer.duplicate()); + } + } + + @Override + public T createInstance() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.createInstance(); + } + } + + @Override + public T copy(T from) { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.copy(from); + } + } + + @Override + public T copy(T from, T reuse) { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.copy(from, reuse); + } + } + + @Override + public int getLength() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.getLength(); + } + } + + @Override + public void serialize(T record, DataOutputView target) throws IOException { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + originalSerializer.serialize(record, target); + } + } + + @Override + public T deserialize(DataInputView source) throws IOException { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.deserialize(source); + } + } + + @Override + public T deserialize(T reuse, DataInputView source) throws IOException { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.deserialize(reuse, source); + } + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + originalSerializer.copy(source, target); + } + } + + @Override + public boolean equals(Object obj) { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return obj == this || + (obj != null && obj.getClass() == getClass() && + originalSerializer.equals(obj)); + } + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof SingleThreadAccessCheckingTypeSerializer; + } + + @Override + public int hashCode() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return originalSerializer.hashCode(); + } + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + return new SingleThreadAccessCheckingTypeSerializerSnapshot<>(this); + } + } + + /** {@link TypeSerializerSnapshot} for {@link SingleThreadAccessCheckingTypeSerializer}. */ + public static class SingleThreadAccessCheckingTypeSerializerSnapshot + extends CompositeTypeSerializerConfigSnapshot { + + SingleThreadAccessCheckingTypeSerializerSnapshot(SingleThreadAccessCheckingTypeSerializer serializerInstance) { + super(serializerInstance); + } + + @Override + public int getVersion() { + return 0; + } + } + + private void writeObject(ObjectOutputStream outputStream) throws IOException { + try (SingleThreadAccessCheck ignored = singleThreadAccessChecker.startSingleThreadAccessCheck()) { + outputStream.defaultWriteObject(); + } + } + + private static class SingleThreadAccessChecker implements Serializable { + private static final long serialVersionUID = 131020282727167064L; + + private transient AtomicReference currentThreadRef = new AtomicReference<>(); + + SingleThreadAccessCheck startSingleThreadAccessCheck() { + assert(currentThreadRef.compareAndSet(null, Thread.currentThread())) : + "The checker has concurrent access from " + currentThreadRef.get(); + return new SingleThreadAccessCheck(currentThreadRef); + } + + private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { + inputStream.defaultReadObject(); + currentThreadRef = new AtomicReference<>(); + } + } + + private static class SingleThreadAccessCheck implements AutoCloseable { + private final AtomicReference currentThreadRef; + + private SingleThreadAccessCheck(AtomicReference currentThreadRef) { + this.currentThreadRef = currentThreadRef; + } + + @Override + public void close() { + assert(currentThreadRef.compareAndSet(Thread.currentThread(), null)) : + "The checker has concurrent access from " + currentThreadRef.get(); + } + } +} diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SlidingWindowCheckMapper.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SlidingWindowCheckMapper.java new file mode 100644 index 0000000000000..8d3242efda5d2 --- /dev/null +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/SlidingWindowCheckMapper.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.ListTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +/** + * This mapper validates sliding event time window. It checks each event belongs to appropriate number of consecutive + * windows. + */ +public class SlidingWindowCheckMapper extends RichFlatMapFunction>, String> { + + private static final long serialVersionUID = -744070793650644485L; + + /** This value state tracks previously seen events with the number of windows they appeared in. */ + private transient ValueState>> eventsSeenSoFar; + + private transient ValueState lastSequenceNumber; + + private final int slideFactor; + + SlidingWindowCheckMapper(int slideFactor) { + this.slideFactor = slideFactor; + } + + @Override + public void open(Configuration parameters) { + ValueStateDescriptor>> previousWindowDescriptor = + new ValueStateDescriptor<>("eventsSeenSoFar", + new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO))); + + eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor); + + ValueStateDescriptor lastSequenceNumberDescriptor = + new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO); + + lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor); + } + + @Override + public void flatMap(Tuple2> value, Collector out) throws Exception { + List> previousWindowValues = Optional.ofNullable(eventsSeenSoFar.value()).orElseGet( + Collections::emptyList); + + List newValues = value.f1; + Optional lastEventInWindow = verifyWindowContiguity(newValues, out); + + Long lastSequenceNumberSeenSoFar = lastSequenceNumber.value(); + List> newWindows = + verifyPreviousOccurences(previousWindowValues, newValues, lastSequenceNumberSeenSoFar, out); + + if (lastEventInWindow.isPresent()) { + updateLastSeenSequenceNumber(lastEventInWindow.get(), lastSequenceNumberSeenSoFar, out); + } + + eventsSeenSoFar.update(newWindows); + } + + private void updateLastSeenSequenceNumber( + Event lastEventInWindow, + Long lastSequenceNumberSeenSoFar, + Collector out) throws IOException { + long lastSequenceNumberInWindow = lastEventInWindow.getSequenceNumber(); + if (lastSequenceNumberSeenSoFar == null || lastSequenceNumberInWindow > lastSequenceNumberSeenSoFar) { + lastSequenceNumber.update(lastSequenceNumberInWindow); + } else if (lastSequenceNumberInWindow < lastSequenceNumberSeenSoFar) { + failWithSequenceNumberDecreased(lastEventInWindow, lastSequenceNumberSeenSoFar, out); + } + } + + private void failWithSequenceNumberDecreased( + Event lastEventInWindow, + Long lastSequenceNumberSeenSoFar, + Collector out) { + out.collect(String.format("Last event in current window (%s) has sequence number lower than seen so far (%d)", + lastEventInWindow, + lastSequenceNumberSeenSoFar)); + } + + /** + * Verifies if all values from previous windows appear in the new one. Returns union of all events seen so far that + * were not seen slideFactor number of times yet. + */ + private List> verifyPreviousOccurences( + List> previousWindowValues, + List newValues, + Long lastSequenceNumberSeenSoFar, + Collector out) { + List> newEventsSeenSoFar = new ArrayList<>(); + List seenEvents = new ArrayList<>(); + + for (Tuple2 windowValue : previousWindowValues) { + if (!newValues.contains(windowValue.f0)) { + failWithEventNotSeenAlertMessage(windowValue, newValues, out); + } else { + seenEvents.add(windowValue.f0); + preserveOrDiscardIfSeenSlideFactorTimes(newEventsSeenSoFar, windowValue); + } + } + + addNotSeenValues(newEventsSeenSoFar, newValues, seenEvents, lastSequenceNumberSeenSoFar, out); + + return newEventsSeenSoFar; + } + + private void addNotSeenValues( + List> newEventsSeenSoFar, + List newValues, + List seenValues, + Long lastSequenceNumberSeenSoFar, + Collector out) { + newValues.stream() + .filter(e -> !seenValues.contains(e)) + .forEach(e -> { + if (lastSequenceNumberSeenSoFar == null || e.getSequenceNumber() > lastSequenceNumberSeenSoFar) { + newEventsSeenSoFar.add(Tuple2.of(e, 1)); + } else { + failWithEventSeenTooManyTimesMessage(e, out); + } + }); + } + + private void failWithEventSeenTooManyTimesMessage(Event e, Collector out) { + out.collect(String.format("Alert: event %s seen more than %d times", e, slideFactor)); + } + + private void preserveOrDiscardIfSeenSlideFactorTimes( + List> newEvenstSeenSoFar, + Tuple2 windowValue) { + int timesSeen = windowValue.f1 + 1; + if (timesSeen != slideFactor) { + newEvenstSeenSoFar.add(Tuple2.of(windowValue.f0, timesSeen)); + } + } + + private void failWithEventNotSeenAlertMessage( + Tuple2 previousWindowValue, + List currentWindowValues, + Collector out) { + out.collect(String.format( + "Alert: event %s did not belong to %d consecutive windows. " + + "Event seen so far %d times.Current window: %s", + previousWindowValue.f0, + slideFactor, + previousWindowValue.f1, + currentWindowValues)); + } + + private Optional verifyWindowContiguity(List newValues, Collector out) { + return newValues.stream() + .sorted(Comparator.comparingLong(Event::getSequenceNumber)) + .reduce((event, event2) -> { + if (event2.getSequenceNumber() - 1 != event.getSequenceNumber()) { + out.collect("Alert: events in window out ouf order!"); + } + + return event2; + }); + } +} diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/TestOperatorEnum.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/TestOperatorEnum.java new file mode 100644 index 0000000000000..4769cedd9ba5e --- /dev/null +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/TestOperatorEnum.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +/** Enum of names and uids of all test operators used in {@link DataStreamAllroundTestProgram}. */ +public enum TestOperatorEnum { + EVENT_SOURCE("EventSource", 1), + KEYED_STATE_OPER_WITH_KRYO_AND_CUSTOM_SER("ArtificalKeyedStateMapper_Kryo_and_Custom_Stateful", 2), + KEYED_STATE_OPER_WITH_AVRO_SER("ArtificalKeyedStateMapper_Avro", 3), + OPERATOR_STATE_OPER("ArtificalOperatorStateMapper", 4), + TIME_WINDOW_OPER("TumblingWindowOperator", 5), + FAILURE_MAPPER_NAME("FailureMapper", 6), + SEMANTICS_CHECK_MAPPER("SemanticsCheckMapper", 7), + SEMANTICS_CHECK_PRINT_SINK("SemanticsCheckPrintSink", 8), + SLIDING_WINDOW_AGG("SlidingWindowOperator", 9), + SLIDING_WINDOW_CHECK_MAPPER("SlidingWindowCheckMapper", 10), + SLIDING_WINDOW_CHECK_PRINT_SINK("SlidingWindowCheckPrintSink", 11), + RESULT_TYPE_QUERYABLE_MAPPER_WITH_CUSTOM_SER("ResultTypeQueryableMapWithCustomStatefulTypeSerializer", 12), + MAPPER_RETURNS_OUT_WITH_CUSTOM_SER("MapReturnsOutputWithCustomStatefulTypeSerializer", 13), + EVENT_IDENTITY_MAPPER("EventIdentityMapper", 14); + + private final String name; + private final String uid; + + TestOperatorEnum(String name, int uid) { + this.name = name; + this.uid = String.format("%04d", uid); + } + + public String getName() { + return name; + } + + public String getUid() { + return uid; + } +} diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/artificialstate/builder/ArtificialStateBuilder.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/artificialstate/builder/ArtificialStateBuilder.java index aed94babff5aa..ac24f3946791f 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/artificialstate/builder/ArtificialStateBuilder.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/artificialstate/builder/ArtificialStateBuilder.java @@ -30,9 +30,9 @@ public abstract class ArtificialStateBuilder implements Serializable { private static final long serialVersionUID = -5887676929924485788L; - protected final String stateName; + final String stateName; - public ArtificialStateBuilder(String stateName) { + ArtificialStateBuilder(String stateName) { this.stateName = stateName; } diff --git a/flink-end-to-end-tests/flink-e2e-test-utils/pom.xml b/flink-end-to-end-tests/flink-e2e-test-utils/pom.xml new file mode 100644 index 0000000000000..fd4942f2b9bef --- /dev/null +++ b/flink-end-to-end-tests/flink-e2e-test-utils/pom.xml @@ -0,0 +1,80 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + + 4.0.0 + + flink-e2e-test-utils + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + com.amazonaws + aws-java-sdk-s3 + 1.11.437 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + S3UtilProgram + package + + shade + + + S3UtilProgram + + + org.apache.flink.streaming.tests.util.s3.S3UtilProgram + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + true + + + + + + diff --git a/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3QueryUtil.java b/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3QueryUtil.java new file mode 100644 index 0000000000000..781a8edd20aca --- /dev/null +++ b/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3QueryUtil.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests.util.s3; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CSVInput; +import com.amazonaws.services.s3.model.CSVOutput; +import com.amazonaws.services.s3.model.CompressionType; +import com.amazonaws.services.s3.model.ExpressionType; +import com.amazonaws.services.s3.model.InputSerialization; +import com.amazonaws.services.s3.model.OutputSerialization; +import com.amazonaws.services.s3.model.SelectObjectContentEvent; +import com.amazonaws.services.s3.model.SelectObjectContentEventStream; +import com.amazonaws.services.s3.model.SelectObjectContentEventVisitor; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.amazonaws.util.IOUtils.copy; + +class S3QueryUtil { + /** Run SQL query over non-compressed CSV file saved in s3 object. */ + static String queryFile( + AmazonS3 s3client, String bucket, String s3file, @SuppressWarnings("SameParameterValue") String query) { + SelectObjectContentRequest request = generateBaseCSVRequest(bucket, s3file, query); + final AtomicBoolean isResultComplete = new AtomicBoolean(false); + String res; + try (SelectObjectContentResult result = s3client.selectObjectContent(request); + SelectObjectContentEventStream payload = result.getPayload(); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + InputStream resultInputStream = payload.getRecordsInputStream( + new SelectObjectContentEventVisitor() { + @Override + public void visit(SelectObjectContentEvent.EndEvent event) { + isResultComplete.set(true); + } + } + ); + copy(resultInputStream, out); + res = out.toString().trim(); + } catch (Throwable e) { + System.out.println("SQL query failure"); + throw new RuntimeException("SQL query failure", e); + } + /* + * The End Event indicates all matching records have been transmitted. + * If the End Event is not received, the results may be incomplete. + */ + if (!isResultComplete.get()) { + throw new RuntimeException("S3 Select request was incomplete as End Event was not received."); + } + return res; + } + + private static SelectObjectContentRequest generateBaseCSVRequest(String bucket, String key, String query) { + SelectObjectContentRequest request = new SelectObjectContentRequest(); + request.setBucketName(bucket); + request.setKey(key); + request.setExpression(query); + request.setExpressionType(ExpressionType.SQL); + + InputSerialization inputSerialization = new InputSerialization(); + inputSerialization.setCsv(new CSVInput()); + inputSerialization.setCompressionType(CompressionType.NONE); + request.setInputSerialization(inputSerialization); + + OutputSerialization outputSerialization = new OutputSerialization(); + outputSerialization.setCsv(new CSVOutput()); + request.setOutputSerialization(outputSerialization); + + return request; + } +} diff --git a/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3UtilProgram.java b/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3UtilProgram.java new file mode 100644 index 0000000000000..70cb55854c573 --- /dev/null +++ b/flink-end-to-end-tests/flink-e2e-test-utils/src/main/java/org/apache/flink/streaming/tests/util/s3/S3UtilProgram.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests.util.s3; + +import org.apache.flink.api.java.utils.ParameterTool; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.KeyFilter; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * S3 utilities. + * + *

    Usage: java -jar S3UtilProgram.jar args. + * + *

    Note: {@code S3UtilProgram.Action.lineNumber*} actions are applicable only + * to valid non-compressed CSV comma separated files. + * + *

    Program parameters: + *

      + *
    • action (string, required): Action to perform, see {@link S3UtilProgram.Action}.
      • + *
    • bucket (string, required): Bucket where s3 objects reside.
      • + *
    • s3file (string, required for single object actions): s3 object key.
      • + *
    • s3prefix (string, required for actions over objects grouped by key prefix): s3 key prefix.
      • + *
    • s3filePrefix (string, optional for downloadByFullPathAndFileNamePrefix or numberOfLinesInFilesWithFullAndNamePrefix): + * s3 file name prefix w/o directory to filter files by name.
      • + *
    • localFile (string, required for single file actions): local file path.
      • + *
    • localFolder (string, required for actions over folders): local folder path.
      • + *
    • parallelism (int, default 10): parallelism for parallelizable actions + * (e.g. {@code numberOfLinesInFilesWithFullAndNamePrefix}).
      • + *
    + */ +class S3UtilProgram { + enum Action { + listByFullPathPrefix, + downloadFile, + downloadByFullPathAndFileNamePrefix, + deleteFile, + deleteByFullPathPrefix, + numberOfLinesInFile, + numberOfLinesInFilesWithFullAndNamePrefix + } + + private static final Map> handlers; + static { + Map> handlersMutable = new HashMap<>(); + handlersMutable.put(Action.listByFullPathPrefix, S3UtilProgram::listByFullPathPrefix); + handlersMutable.put(Action.downloadFile, S3UtilProgram::downloadFile); + handlersMutable.put(Action.downloadByFullPathAndFileNamePrefix, + S3UtilProgram::downloadByFullPathAndFileNamePrefix); + handlersMutable.put(Action.deleteFile, S3UtilProgram::deleteFile); + handlersMutable.put(Action.deleteByFullPathPrefix, S3UtilProgram::deleteByFullPathPrefix); + handlersMutable.put(Action.numberOfLinesInFile, S3UtilProgram::numberOfLinesInFile); + handlersMutable.put(Action.numberOfLinesInFilesWithFullAndNamePrefix, + S3UtilProgram::numberOfLinesInFilesWithFullAndNamePrefix); + handlers = Collections.unmodifiableMap(handlersMutable); + } + + private static final String countQuery = "select count(*) from s3object"; + + public static void main(String[] args) { + final ParameterTool params = ParameterTool.fromArgs(args); + final Action action = Action.valueOf(params.getRequired("action")); + handlers.get(action).accept(params); + } + + private static void listByFullPathPrefix(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3prefix = params.getRequired("s3prefix"); + listByFullPathPrefix(bucket, s3prefix).forEach(System.out::println); + } + + private static List listByFullPathPrefix(final String bucket, final String s3prefix) { + return AmazonS3ClientBuilder.defaultClient().listObjects(bucket, s3prefix).getObjectSummaries() + .stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()); + } + + private static void downloadFile(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3file = params.getRequired("s3file"); + final String localFile = params.getRequired("localFile"); + TransferManager tx = TransferManagerBuilder.defaultTransferManager(); + try { + tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); + } catch (InterruptedException e) { + System.out.println("Transfer interrupted"); + } finally { + tx.shutdownNow(); + } + } + + private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3prefix = params.getRequired("s3prefix"); + final String localFolder = params.getRequired("localFolder"); + final String s3filePrefix = params.get("s3filePrefix", ""); + TransferManager tx = TransferManagerBuilder.defaultTransferManager(); + Predicate keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); + KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : + objectSummary -> keyPredicate.test(objectSummary.getKey()); + try { + tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); + } catch (InterruptedException e) { + System.out.println("Transfer interrupted"); + } finally { + tx.shutdownNow(); + } + } + + private static Predicate getKeyFilterByFileNamePrefix(String s3filePrefix) { + if (s3filePrefix.isEmpty()) { + return key -> true; + } else { + return key -> { + String[] parts = key.split("/"); + String fileName = parts[parts.length - 1]; + return fileName.startsWith(s3filePrefix); + }; + } + } + + private static void deleteFile(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3file = params.getRequired("s3file"); + AmazonS3ClientBuilder.defaultClient().deleteObject(bucket, s3file); + } + + private static void deleteByFullPathPrefix(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3prefix = params.getRequired("s3prefix"); + String[] keys = listByFullPathPrefix(bucket, s3prefix).toArray(new String[] {}); + if (keys.length > 0) { + DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(keys); + AmazonS3ClientBuilder.defaultClient().deleteObjects(request); + } + } + + private static void numberOfLinesInFile(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3file = params.getRequired("s3file"); + AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient(); + System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery)); + s3client.shutdown(); + } + + private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) { + final String bucket = params.getRequired("bucket"); + final String s3prefix = params.getRequired("s3prefix"); + final String s3filePrefix = params.get("s3filePrefix", ""); + int parallelism = params.getInt("parallelism", 10); + + List files = listByFullPathPrefix(bucket, s3prefix); + + ExecutorService executor = Executors.newFixedThreadPool(parallelism); + AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient(); + List> requests = + submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix); + int count = waitAndComputeTotalLineCountResult(requests); + + executor.shutdownNow(); + s3client.shutdown(); + System.out.print(count); + } + + private static List> submitLineCountingRequestsForFilesAsync( + ExecutorService executor, AmazonS3 s3client, String bucket, List files, String s3filePrefix) { + List> requests = new ArrayList<>(); + Predicate keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); + files.forEach(file -> { + if (keyPredicate.test(file)) { + CompletableFuture result = new CompletableFuture<>(); + executor.execute(() -> + result.complete(Integer.parseInt(S3QueryUtil.queryFile(s3client, bucket, file, countQuery)))); + requests.add(result); + } + }); + return requests; + } + + private static int waitAndComputeTotalLineCountResult(List> requests) { + int count = 0; + for (CompletableFuture result : requests) { + try { + count += result.get(); + } catch (Throwable e) { + System.out.println("Failed count lines"); + e.printStackTrace(); + } + } + return count; + } +} diff --git a/flink-end-to-end-tests/flink-elasticsearch6-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch6SinkExample.java b/flink-end-to-end-tests/flink-elasticsearch6-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch6SinkExample.java index e813c2995f5f7..6533ea220b482 100644 --- a/flink-end-to-end-tests/flink-elasticsearch6-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch6SinkExample.java +++ b/flink-end-to-end-tests/flink-elasticsearch6-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch6SinkExample.java @@ -23,11 +23,13 @@ import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; import org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink; import org.apache.flink.util.Collector; import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Requests; @@ -77,6 +79,9 @@ public void flatMap(Long value, Collector> out) { indexer.add(createUpdateRequest(element, parameterTool)); }); + esSinkBuilder.setFailureHandler( + new CustomFailureHandler(parameterTool.getRequired("index"), parameterTool.getRequired("type"))); + // this instructs the sink to emit after every element, otherwise they would be buffered esSinkBuilder.setBulkFlushMaxActions(1); @@ -85,13 +90,54 @@ public void flatMap(Long value, Collector> out) { env.execute("Elasticsearch 6.x end to end sink test example"); } + private static class CustomFailureHandler implements ActionRequestFailureHandler { + + private static final long serialVersionUID = 942269087742453482L; + + private final String index; + private final String type; + + CustomFailureHandler(String index, String type) { + this.index = index; + this.type = type; + } + + @Override + public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable { + if (action instanceof IndexRequest) { + Map json = new HashMap<>(); + json.put("data", ((IndexRequest) action).source()); + + indexer.add( + Requests.indexRequest() + .index(index) + .type(type) + .id(((IndexRequest) action).id()) + .source(json)); + } else { + throw new IllegalStateException("unexpected"); + } + } + } + private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map json = new HashMap<>(); json.put("data", element); + String index; + String type; + + if (element.startsWith("message #15")) { + index = ":intentional invalid index:"; + type = ":intentional invalid type:"; + } else { + index = parameterTool.getRequired("index"); + type = parameterTool.getRequired("type"); + } + return Requests.indexRequest() - .index(parameterTool.getRequired("index")) - .type(parameterTool.getRequired("type")) + .index(index) + .type(type) .id(element) .source(json); } diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-common/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-common/pom.xml new file mode 100644 index 0000000000000..eece2d597bfad --- /dev/null +++ b/flink-end-to-end-tests/flink-end-to-end-tests-common/pom.xml @@ -0,0 +1,63 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + + + 4.0.0 + + flink-end-to-end-tests-common + 1.7-SNAPSHOT + + + + org.apache.flink + flink-core + ${project.version} + + + com.squareup.okhttp3 + okhttp + 3.7.0 + + + org.apache.flink + flink-shaded-jackson + + + junit + junit + ${junit.version} + compile + + + + org.apache.flink + flink-dist_${scala.binary.version} + ${project.version} + provided + + + + diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosablePath.java b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosablePath.java new file mode 100644 index 0000000000000..0098889ead619 --- /dev/null +++ b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosablePath.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.tests.util; + +import org.apache.flink.util.FileUtils; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Utility class to delete a given {@link Path} when exiting a try-with-resources statement. + */ +public final class AutoClosablePath implements AutoCloseable { + + private final Path path; + + public AutoClosablePath(final Path path) { + Preconditions.checkNotNull(path, "Path must not be null."); + Preconditions.checkArgument(path.isAbsolute(), "Path must be absolute."); + this.path = path; + } + + @Override + public void close() throws IOException { + FileUtils.deleteFileOrDirectory(path.toFile()); + } +} diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosableProcess.java b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosableProcess.java new file mode 100644 index 0000000000000..02359302e13f1 --- /dev/null +++ b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/AutoClosableProcess.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.tests.util; + +import org.apache.flink.util.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Utility class to terminate a given {@link Process} when exiting a try-with-resources statement. + */ +public class AutoClosableProcess implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(AutoClosableProcess.class); + + private final Process process; + + public AutoClosableProcess(final Process process) { + Preconditions.checkNotNull(process); + this.process = process; + } + + public static AutoClosableProcess runNonBlocking(String step, String... commands) throws IOException { + LOG.info("Step Started: " + step); + Process process = new ProcessBuilder() + .command(commands) + .inheritIO() + .start(); + return new AutoClosableProcess(process); + } + + public static void runBlocking(String step, String... commands) throws IOException { + runBlocking(step, Duration.ofSeconds(30), commands); + } + + public static void runBlocking(String step, Duration timeout, String... commands) throws IOException { + LOG.info("Step started: " + step); + Process process = new ProcessBuilder() + .command(commands) + .inheritIO() + .start(); + + try (AutoClosableProcess autoProcess = new AutoClosableProcess(process)) { + final boolean success = process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS); + if (!success) { + throw new TimeoutException(); + } + } catch (TimeoutException | InterruptedException e) { + throw new RuntimeException(step + " failed due to timeout."); + } + LOG.info("Step complete: " + step); + } + + @Override + public void close() throws IOException { + if (process.isAlive()) { + process.destroy(); + try { + process.waitFor(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/CommandLineWrapper.java b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/CommandLineWrapper.java new file mode 100644 index 0000000000000..50fd2f81f0256 --- /dev/null +++ b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/CommandLineWrapper.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.tests.util; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility class for setting up command-line tool usages in a readable fashion. + */ +public enum CommandLineWrapper { + ; + + public static WGetBuilder wget(String url) { + return new WGetBuilder(url); + } + + /** + * Wrapper around wget used for downloading files. + */ + public static final class WGetBuilder { + + private final String url; + private Path targetDir; + + WGetBuilder(String url) { + this.url = url; + } + + public WGetBuilder targetDir(Path dir) { + this.targetDir = dir; + return this; + } + + public String[] build() { + final List commandsList = new ArrayList<>(5); + commandsList.add("wget"); + commandsList.add("-q"); // silent + //commandsList.add("--show-progress"); // enable progress bar + if (targetDir != null) { + commandsList.add("-P"); + commandsList.add(targetDir.toAbsolutePath().toString()); + } + commandsList.add(url); + return commandsList.toArray(new String[commandsList.size()]); + } + } + + public static SedBuilder sed(final String command, final Path file) { + return new SedBuilder(command, file); + } + + /** + * Wrapper around sed used for processing text. + */ + public static final class SedBuilder { + + private final String command; + private final Path file; + + private boolean inPlace = false; + + SedBuilder(final String command, final Path file) { + this.command = command; + this.file = file; + } + + public SedBuilder inPlace() { + inPlace = true; + return this; + } + + public String[] build() { + final List commandsList = new ArrayList<>(5); + commandsList.add("sed"); + if (inPlace) { + commandsList.add("-i"); + } + commandsList.add("-e"); + commandsList.add(command); + commandsList.add(file.toAbsolutePath().toString()); + return commandsList.toArray(new String[commandsList.size()]); + } + } + + public static TarBuilder tar(final Path file) { + return new TarBuilder(file); + } + + /** + * Wrapper around tar used for extracting .tar archives. + */ + public static final class TarBuilder { + + private final Path file; + private boolean zipped = false; + private boolean extract = false; + private Path targetDir; + + public TarBuilder(final Path file) { + this.file = file; + } + + public TarBuilder zipped() { + zipped = true; + return this; + } + + public TarBuilder extract() { + extract = true; + return this; + } + + public TarBuilder targetDir(final Path dir) { + targetDir = dir; + return this; + } + + public String[] build() { + final List commandsList = new ArrayList<>(4); + commandsList.add("tar"); + if (zipped) { + commandsList.add("-z"); + } + if (extract) { + commandsList.add("-x"); + } + if (targetDir != null) { + commandsList.add("--directory"); + commandsList.add(targetDir.toAbsolutePath().toString()); + } + commandsList.add("-f"); + commandsList.add(file.toAbsolutePath().toString()); + return commandsList.toArray(new String[commandsList.size()]); + } + } +} diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/FlinkDistribution.java b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/FlinkDistribution.java new file mode 100644 index 0000000000000..7031d88c89013 --- /dev/null +++ b/flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/FlinkDistribution.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.tests.util; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.configuration.UnmodifiableConfiguration; +import org.apache.flink.util.ExceptionUtils; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.Assert; +import org.junit.rules.ExternalResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A wrapper around a Flink distribution. + */ +public final class FlinkDistribution extends ExternalResource { + + private static final Logger LOG = LoggerFactory.getLogger(FlinkDistribution.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final Path FLINK_CONF_YAML = Paths.get("flink-conf.yaml"); + private static final Path FLINK_CONF_YAML_BACKUP = Paths.get("flink-conf.yaml.bak"); + + private final List filesToDelete = new ArrayList<>(4); + + private final Path opt; + private final Path lib; + private final Path conf; + private final Path log; + private final Path bin; + + private Configuration defaultConfig; + + public FlinkDistribution() { + final String distDirProperty = System.getProperty("distDir"); + if (distDirProperty == null) { + Assert.fail("The distDir property was not set. You can set it when running maven via -DdistDir= ."); + } + final Path flinkDir = Paths.get(distDirProperty); + bin = flinkDir.resolve("bin"); + opt = flinkDir.resolve("opt"); + lib = flinkDir.resolve("lib"); + conf = flinkDir.resolve("conf"); + log = flinkDir.resolve("log"); + } + + @Override + protected void before() throws IOException { + defaultConfig = new UnmodifiableConfiguration(GlobalConfiguration.loadConfiguration(conf.toAbsolutePath().toString())); + final Path originalConfig = conf.resolve(FLINK_CONF_YAML); + final Path backupConfig = conf.resolve(FLINK_CONF_YAML_BACKUP); + Files.copy(originalConfig, backupConfig); + filesToDelete.add(new AutoClosablePath(backupConfig)); + } + + @Override + protected void after() { + try { + stopFlinkCluster(); + } catch (IOException e) { + LOG.error("Failure while shutting down Flink cluster.", e); + } + + final Path originalConfig = conf.resolve(FLINK_CONF_YAML); + final Path backupConfig = conf.resolve(FLINK_CONF_YAML_BACKUP); + + try { + Files.move(backupConfig, originalConfig, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + LOG.error("Failed to restore flink-conf.yaml", e); + } + + for (AutoCloseable fileToDelete : filesToDelete) { + try { + fileToDelete.close(); + } catch (Exception e) { + LOG.error("Failure while cleaning up file.", e); + } + } + } + + public void startFlinkCluster() throws IOException { + AutoClosableProcess.runBlocking("Start Flink cluster", bin.resolve("start-cluster.sh").toAbsolutePath().toString()); + + final OkHttpClient client = new OkHttpClient(); + + final Request request = new Request.Builder() + .get() + .url("http://localhost:8081/taskmanagers") + .build(); + + Exception reportedException = null; + for (int retryAttempt = 0; retryAttempt < 30; retryAttempt++) { + try (Response response = client.newCall(request).execute()) { + if (response.isSuccessful()) { + final String json = response.body().string(); + final JsonNode taskManagerList = OBJECT_MAPPER.readTree(json) + .get("taskmanagers"); + + if (taskManagerList != null && taskManagerList.size() > 0) { + LOG.info("Dispatcher REST endpoint is up."); + return; + } + } + } catch (IOException ioe) { + reportedException = ExceptionUtils.firstOrSuppressed(ioe, reportedException); + } + + LOG.info("Waiting for dispatcher REST endpoint to come up..."); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + reportedException = ExceptionUtils.firstOrSuppressed(e, reportedException); + } + } + throw new AssertionError("Dispatcher REST endpoint did not start in time.", reportedException); + } + + public void stopFlinkCluster() throws IOException { + AutoClosableProcess.runBlocking("Stop Flink Cluster", bin.resolve("stop-cluster.sh").toAbsolutePath().toString()); + } + + public void copyOptJarsToLib(String jarNamePrefix) throws FileNotFoundException, IOException { + final Optional reporterJarOptional = Files.walk(opt) + .filter(path -> path.getFileName().toString().startsWith(jarNamePrefix)) + .findFirst(); + if (reporterJarOptional.isPresent()) { + final Path optReporterJar = reporterJarOptional.get(); + final Path libReporterJar = lib.resolve(optReporterJar.getFileName()); + Files.copy(optReporterJar, libReporterJar); + filesToDelete.add(new AutoClosablePath(libReporterJar)); + } else { + throw new FileNotFoundException("No jar could be found matching the pattern " + jarNamePrefix + "."); + } + } + + public void appendConfiguration(Configuration config) throws IOException { + final Configuration mergedConfig = new Configuration(); + mergedConfig.addAll(defaultConfig); + mergedConfig.addAll(config); + + final List configurationLines = mergedConfig.toMap().entrySet().stream() + .map(entry -> entry.getKey() + ": " + entry.getValue()) + .collect(Collectors.toList()); + + Files.write(conf.resolve("flink-conf.yaml"), configurationLines); + } + + public Stream searchAllLogs(Pattern pattern, Function matchProcessor) throws IOException { + final List matches = new ArrayList<>(2); + + try (Stream logFilesStream = Files.list(log)) { + final Iterator logFiles = logFilesStream.iterator(); + while (logFiles.hasNext()) { + final Path logFile = logFiles.next(); + if (!logFile.getFileName().toString().endsWith(".log")) { + // ignore logs for previous runs that have a number suffix + continue; + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logFile.toFile()), StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + matches.add(matchProcessor.apply(matcher)); + } + } + } + } + } + return matches.stream(); + } +} diff --git a/flink-metrics/flink-metrics-ganglia/pom.xml b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/pom.xml similarity index 62% rename from flink-metrics/flink-metrics-ganglia/pom.xml rename to flink-end-to-end-tests/flink-heavy-deployment-stress-test/pom.xml index 1cec50e6c1a26..b3384defe1682 100644 --- a/flink-metrics/flink-metrics-ganglia/pom.xml +++ b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/pom.xml @@ -21,77 +21,61 @@ under the License. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - + org.apache.flink - flink-metrics + flink-end-to-end-tests 1.7-SNAPSHOT .. - flink-metrics-ganglia - flink-metrics-ganglia + flink-heavy-deployment-stress-test + flink-heavy-deployment-stress-test + jar org.apache.flink - flink-annotations + flink-streaming-java_${scala.binary.version} ${project.version} - provided - org.apache.flink - flink-metrics-core + flink-datastream-allround-test ${project.version} - provided - org.apache.flink - flink-metrics-dropwizard + flink-test-utils_${scala.binary.version} ${project.version} - - - - info.ganglia.gmetric4j - gmetric4j - 1.0.7 - - - - io.dropwizard.metrics - metrics-core - ${metrics.version} - - - - io.dropwizard.metrics - metrics-ganglia - ${metrics.version} + compile - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - + org.apache.maven.plugins + maven-shade-plugin - make-assembly + HeavyDeploymentStressTestProgram package - single + shade + + HeavyDeploymentStressTestProgram + + + org.apache.flink.deployment.HeavyDeploymentStressTestProgram + + + + diff --git a/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java new file mode 100644 index 0000000000000..d65583f98afce --- /dev/null +++ b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.deployment; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.OperatorStateStore; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.runtime.state.CheckpointListener; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.DiscardingSink; +import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; +import org.apache.flink.util.Preconditions; + +import static org.apache.flink.streaming.tests.DataStreamAllroundTestJobFactory.setupEnvironment; + + +/** + * End-to-end test for heavy deployment descriptors. This test creates a heavy deployment by producing inflated meta + * data for the source's operator state. The state is registered as union state and will be multiplied in deployment. + */ +public class HeavyDeploymentStressTestProgram { + + private static final ConfigOption NUM_LIST_STATES_PER_OP = ConfigOptions + .key("heavy_deployment_test.num_list_states_per_op") + .defaultValue(100); + + private static final ConfigOption NUM_PARTITIONS_PER_LIST_STATE = ConfigOptions + .key("heavy_deployment_test.num_partitions_per_list_state") + .defaultValue(100); + + public static void main(String[] args) throws Exception { + + final ParameterTool pt = ParameterTool.fromArgs(args); + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + setupEnvironment(env, pt); + + final int numStates = + pt.getInt(NUM_LIST_STATES_PER_OP.key(), NUM_LIST_STATES_PER_OP.defaultValue()); + final int numPartitionsPerState = + pt.getInt(NUM_PARTITIONS_PER_LIST_STATE.key(), NUM_PARTITIONS_PER_LIST_STATE.defaultValue()); + + Preconditions.checkState(env.getCheckpointInterval() > 0L, "Checkpointing must be enabled for this test!"); + + env.addSource(new SimpleEndlessSourceWithBloatedState(numStates, numPartitionsPerState)).setParallelism(env.getParallelism()) + .addSink(new DiscardingSink<>()).setParallelism(1); + + env.execute("HeavyDeploymentStressTestProgram"); + } + + /** + * Source with dummy operator state that results in inflated meta data. + */ + static class SimpleEndlessSourceWithBloatedState extends RichParallelSourceFunction + implements CheckpointedFunction, CheckpointListener { + + private static final long serialVersionUID = 1L; + + private final int numListStates; + private final int numPartitionsPerListState; + + private transient volatile boolean isRunning; + + /** Flag to induce failure after we have a valid checkpoint. */ + private transient volatile boolean readyToFail; + + SimpleEndlessSourceWithBloatedState(int numListStates, int numPartitionsPerListState) { + this.numListStates = numListStates; + this.numPartitionsPerListState = numPartitionsPerListState; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) { + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + + readyToFail = false; + + if (context.isRestored()) { + isRunning = false; + } else { + isRunning = true; + + OperatorStateStore operatorStateStore = context.getOperatorStateStore(); + for (int i = 0; i < numListStates; ++i) { + + ListStateDescriptor listStateDescriptor = + new ListStateDescriptor<>("test-list-state-" + i, String.class); + + ListState unionListState = + operatorStateStore.getUnionListState(listStateDescriptor); + + for (int j = 0; j < numPartitionsPerListState; ++j) { + unionListState.add(String.valueOf(j)); + } + } + } + } + + @Override + public void run(SourceContext ctx) throws Exception { + while (isRunning) { + + if (readyToFail && getRuntimeContext().getIndexOfThisSubtask() == 0) { + throw new Exception("Artificial failure."); + } + + synchronized (ctx.getCheckpointLock()) { + ctx.collect("test-element"); + } + + Thread.sleep(1); + } + } + + @Override + public void cancel() { + this.isRunning = false; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) { + readyToFail = true; + } + } +} diff --git a/flink-end-to-end-tests/flink-metrics-availability-test/pom.xml b/flink-end-to-end-tests/flink-metrics-availability-test/pom.xml new file mode 100644 index 0000000000000..9027498c5523a --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-availability-test/pom.xml @@ -0,0 +1,93 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + + 4.0.0 + + flink-metrics-availability-test + + + + org.apache.flink + flink-core + ${project.version} + test + + + org.apache.flink + flink-runtime_${scala.binary.version} + ${project.version} + test + + + org.apache.flink + flink-end-to-end-tests-common + ${project.version} + test + + + org.apache.flink + flink-test-utils-junit + ${project.version} + test + + + + + + e2e-metric-availability + + + e2e-metrics + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + e2e-metric-availability + integration-test + + test + + + + **/*ITCase.* + + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-metrics-availability-test/src/test/java/org/pache/flink/metrics/tests/MetricsAvailabilityITCase.java b/flink-end-to-end-tests/flink-metrics-availability-test/src/test/java/org/pache/flink/metrics/tests/MetricsAvailabilityITCase.java new file mode 100644 index 0000000000000..28a21354e5d27 --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-availability-test/src/test/java/org/pache/flink/metrics/tests/MetricsAvailabilityITCase.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pache.flink.metrics.tests; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; +import org.apache.flink.runtime.rest.RestClient; +import org.apache.flink.runtime.rest.RestClientConfiguration; +import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; +import org.apache.flink.runtime.rest.messages.EmptyRequestBody; +import org.apache.flink.runtime.rest.messages.job.metrics.JobManagerMetricsHeaders; +import org.apache.flink.runtime.rest.messages.job.metrics.JobManagerMetricsMessageParameters; +import org.apache.flink.runtime.rest.messages.job.metrics.MetricCollectionResponseBody; +import org.apache.flink.runtime.rest.messages.job.metrics.TaskManagerMetricsHeaders; +import org.apache.flink.runtime.rest.messages.job.metrics.TaskManagerMetricsMessageParameters; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagersHeaders; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagersInfo; +import org.apache.flink.tests.util.FlinkDistribution; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.function.SupplierWithException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * End-to-end test for the availability of metrics. + */ +public class MetricsAvailabilityITCase extends TestLogger { + + private static final String HOST = "localhost"; + private static final int PORT = 8081; + + @Rule + public final FlinkDistribution dist = new FlinkDistribution(); + + @Nullable + private static ScheduledExecutorService scheduledExecutorService = null; + + @BeforeClass + public static void startExecutor() { + scheduledExecutorService = Executors.newScheduledThreadPool(4); + } + + @AfterClass + public static void shutdownExecutor() { + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdown(); + } + } + + @Test + public void testReporter() throws Exception { + dist.startFlinkCluster(); + + final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), scheduledExecutorService); + + checkJobManagerMetricAvailability(restClient); + + final Collection taskManagerIds = getTaskManagerIds(restClient); + + for (final ResourceID taskManagerId : taskManagerIds) { + checkTaskManagerMetricAvailability(restClient, taskManagerId); + } + } + + private static void checkJobManagerMetricAvailability(final RestClient restClient) throws Exception { + final JobManagerMetricsHeaders headers = JobManagerMetricsHeaders.getInstance(); + final JobManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); + parameters.metricsFilterParameter.resolve(Collections.singletonList("numRegisteredTaskManagers")); + + fetchMetric(() -> + restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), + getMetricNamePredicate("numRegisteredTaskManagers")); + } + + private static Collection getTaskManagerIds(final RestClient restClient) throws Exception { + final TaskManagersHeaders headers = TaskManagersHeaders.getInstance(); + + final TaskManagersInfo response = fetchMetric(() -> + restClient.sendRequest( + HOST, + PORT, + headers, + EmptyMessageParameters.getInstance(), + EmptyRequestBody.getInstance()), + taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty()); + + return response.getTaskManagerInfos().stream() + .map(TaskManagerInfo::getResourceId) + .collect(Collectors.toList()); + } + + private static void checkTaskManagerMetricAvailability(final RestClient restClient, final ResourceID taskManagerId) throws Exception { + final TaskManagerMetricsHeaders headers = TaskManagerMetricsHeaders.getInstance(); + final TaskManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); + parameters.taskManagerIdParameter.resolve(taskManagerId); + parameters.metricsFilterParameter.resolve(Collections.singletonList("Status.Network.TotalMemorySegments")); + + fetchMetric(() -> + restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), + getMetricNamePredicate("Status.Network.TotalMemorySegments")); + } + + private static X fetchMetric(final SupplierWithException, IOException> clientOperation, final Predicate predicate) throws InterruptedException, ExecutionException, TimeoutException { + final CompletableFuture responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> { + try { + return clientOperation.get(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + Time.seconds(1), + Deadline.fromNow(Duration.ofSeconds(5)), + predicate, + new ScheduledExecutorServiceAdapter(scheduledExecutorService)); + + return responseFuture.get(30, TimeUnit.SECONDS); + } + + private static Predicate getMetricNamePredicate(final String metricName) { + return response -> response.getMetrics().stream().anyMatch(metric -> metric.getId().equals(metricName)); + } +} diff --git a/flink-end-to-end-tests/flink-metrics-availability-test/src/test/resources/log4j-test.properties b/flink-end-to-end-tests/flink-metrics-availability-test/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000..f7425cd14c098 --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-availability-test/src/test/resources/log4j-test.properties @@ -0,0 +1,27 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +log4j.rootLogger=INFO, testlogger + +# A1 is set to be a ConsoleAppender. +log4j.appender.testlogger=org.apache.log4j.ConsoleAppender +log4j.appender.testlogger.target = System.err +log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout +log4j.appender.testlogger.layout.ConversionPattern=%m%n diff --git a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/pom.xml b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/pom.xml new file mode 100644 index 0000000000000..4163d6a69eacf --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/pom.xml @@ -0,0 +1,104 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + + 4.0.0 + + flink-metrics-reporter-prometheus-test + + + + org.apache.flink + flink-core + ${project.version} + test + + + org.apache.flink + flink-metrics-prometheus_${scala.binary.version} + ${project.version} + test + + + com.squareup.okhttp3 + okhttp + 3.7.0 + test + + + org.apache.flink + flink-shaded-jackson + test + + + org.apache.flink + flink-end-to-end-tests-common + ${project.version} + test + + + org.apache.flink + flink-test-utils-junit + ${project.version} + test + + + + + + e2e-prometheus + + + e2e-metrics + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + e2e-prometheus + integration-test + + test + + + + **/*ITCase.* + + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java new file mode 100644 index 0000000000000..269754eaf6998 --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/java/org/apache/flink/metrics/prometheus/tests/PrometheusReporterEndToEndITCase.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.metrics.prometheus.tests; + +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.prometheus.PrometheusReporter; +import org.apache.flink.tests.util.AutoClosableProcess; +import org.apache.flink.tests.util.CommandLineWrapper; +import org.apache.flink.tests.util.FlinkDistribution; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.OperatingSystem; +import org.apache.flink.util.TestLogger; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.apache.flink.tests.util.AutoClosableProcess.runBlocking; +import static org.apache.flink.tests.util.AutoClosableProcess.runNonBlocking; + +/** + * End-to-end test for the PrometheusReporter. + */ +public class PrometheusReporterEndToEndITCase extends TestLogger { + + private static final Logger LOG = LoggerFactory.getLogger(PrometheusReporterEndToEndITCase.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final String PROMETHEUS_VERSION = "2.4.3"; + private static final String PROMETHEUS_FILE_NAME; + + static { + final String base = "prometheus-" + PROMETHEUS_VERSION + '.'; + switch (OperatingSystem.getCurrentOperatingSystem()) { + case MAC_OS: + PROMETHEUS_FILE_NAME = base + "darwin-amd64"; + break; + case WINDOWS: + PROMETHEUS_FILE_NAME = base + "windows-amd64"; + break; + default: + PROMETHEUS_FILE_NAME = base + "linux-amd64"; + break; + } + } + + private static final Pattern LOG_REPORTER_PORT_PATTERN = Pattern.compile(".*Started PrometheusReporter HTTP server on port ([0-9]+).*"); + + @BeforeClass + public static void checkOS() { + Assume.assumeFalse("This test does not run on Windows.", OperatingSystem.isWindows()); + } + + @Rule + public final FlinkDistribution dist = new FlinkDistribution(); + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void testReporter() throws Exception { + dist.copyOptJarsToLib("flink-metrics-prometheus"); + + final Configuration config = new Configuration(); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "prom." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, PrometheusReporter.class.getCanonicalName()); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "prom.port", "9000-9100"); + + dist.appendConfiguration(config); + + final Path tmpPrometheusDir = tmp.newFolder().toPath().resolve("prometheus"); + final Path prometheusArchive = tmpPrometheusDir.resolve(PROMETHEUS_FILE_NAME + ".tar.gz"); + final Path prometheusBinDir = tmpPrometheusDir.resolve(PROMETHEUS_FILE_NAME); + final Path prometheusConfig = prometheusBinDir.resolve("prometheus.yml"); + final Path prometheusBinary = prometheusBinDir.resolve("prometheus"); + Files.createDirectory(tmpPrometheusDir); + + runBlocking( + "Download of Prometheus", + Duration.ofMinutes(5), + CommandLineWrapper + .wget("https://github.com/prometheus/prometheus/releases/download/v" + PROMETHEUS_VERSION + '/' + prometheusArchive.getFileName()) + .targetDir(tmpPrometheusDir) + .build()); + + runBlocking("Extraction of Prometheus archive", + CommandLineWrapper + .tar(prometheusArchive) + .extract() + .zipped() + .targetDir(tmpPrometheusDir) + .build()); + + runBlocking("Set Prometheus scrape interval", + CommandLineWrapper + .sed("s/\\(scrape_interval:\\).*/\\1 1s/", prometheusConfig) + .inPlace() + .build()); + + dist.startFlinkCluster(); + + final List ports = dist + .searchAllLogs(LOG_REPORTER_PORT_PATTERN, matcher -> matcher.group(1)) + .map(Integer::valueOf) + .collect(Collectors.toList()); + + final String scrapeTargets = ports.stream() + .map(port -> "'localhost:" + port + "'") + .collect(Collectors.joining(", ")); + + runBlocking("Set Prometheus scrape targets to (" + scrapeTargets + ")", + CommandLineWrapper + .sed("s/\\(targets:\\).*/\\1 [" + scrapeTargets + "]/", prometheusConfig) + .inPlace() + .build()); + + try (AutoClosableProcess prometheus = runNonBlocking( + "Start Prometheus server", + prometheusBinary.toAbsolutePath().toString(), + "--config.file=" + prometheusConfig.toAbsolutePath(), + "--storage.tsdb.path=" + prometheusBinDir.resolve("data").toAbsolutePath())) { + + final OkHttpClient client = new OkHttpClient(); + + checkMetricAvailability(client, "flink_jobmanager_numRegisteredTaskManagers"); + checkMetricAvailability(client, "flink_taskmanager_Status_Network_TotalMemorySegments"); + } + } + + private static void checkMetricAvailability(final OkHttpClient client, final String metric) throws InterruptedException { + final Request jobManagerRequest = new Request.Builder() + .get() + .url("http://localhost:9090/api/v1/query?query=" + metric) + .build(); + + Exception reportedException = null; + for (int x = 0; x < 30; x++) { + try (Response response = client.newCall(jobManagerRequest).execute()) { + if (response.isSuccessful()) { + final String json = response.body().string(); + + // Sample response: + //{ + // "status": "success", + // "data": { + // "resultType": "vector", + // "result": [{ + // "metric": { + // "__name__": "flink_jobmanager_numRegisteredTaskManagers", + // "host": "localhost", + // "instance": "localhost:9000", + // "job": "prometheus" + // }, + // "value": [1540548500.107, "1"] + // }] + // } + //} + OBJECT_MAPPER.readTree(json) + .get("data") + .get("result") + .get(0) + .get("value") + .get(1).asInt(); + // if we reach this point some value for the given metric was reported to prometheus + return; + } else { + LOG.info("Retrieving metric failed. Retrying... " + response.code() + ":" + response.message()); + Thread.sleep(1000); + } + } catch (Exception e) { + reportedException = ExceptionUtils.firstOrSuppressed(e, reportedException); + Thread.sleep(1000); + } + } + throw new AssertionError("Could not retrieve metric " + metric + " from Prometheus.", reportedException); + } +} diff --git a/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/resources/log4j-test.properties b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000..f7425cd14c098 --- /dev/null +++ b/flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/src/test/resources/log4j-test.properties @@ -0,0 +1,27 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +log4j.rootLogger=INFO, testlogger + +# A1 is set to be a ConsoleAppender. +log4j.appender.testlogger=org.apache.log4j.ConsoleAppender +log4j.appender.testlogger.target = System.err +log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout +log4j.appender.testlogger.layout.ConversionPattern=%m%n diff --git a/flink-end-to-end-tests/flink-sql-client-test/pom.xml b/flink-end-to-end-tests/flink-sql-client-test/pom.xml index de1db9e6bf596..5edd8e35ae7a3 100644 --- a/flink-end-to-end-tests/flink-sql-client-test/pom.xml +++ b/flink-end-to-end-tests/flink-sql-client-test/pom.xml @@ -89,6 +89,14 @@ under the License. sql-jar provided + + + org.apache.flink + flink-connector-kafka_${scala.binary.version} + ${project.version} + sql-jar + provided + org.apache.flink @@ -106,7 +114,7 @@ under the License. as we neither access nor package the kafka dependencies --> org.apache.kafka kafka-clients - 0.11.0.2 + 2.0.1 @@ -176,14 +184,20 @@ under the License. sql-jar jar - + + + org.apache.flink + flink-connector-kafka_${scala.binary.version} + ${project.version} + sql-jar + jar + org.apache.flink flink-connector-elasticsearch6_${scala.binary.version} diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/DataStreamStateTTLTestProgram.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/DataStreamStateTTLTestProgram.java index 3b2e4746b62df..1a572f314c52f 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/DataStreamStateTTLTestProgram.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/DataStreamStateTTLTestProgram.java @@ -23,6 +23,7 @@ import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.PrintSinkFunction; @@ -74,6 +75,8 @@ public static void main(String[] args) throws Exception { setupEnvironment(env, pt); + final MonotonicTTLTimeProvider ttlTimeProvider = setBackendWithCustomTTLTimeProvider(env); + int keySpace = pt.getInt(UPDATE_GENERATOR_SRC_KEYSPACE.key(), UPDATE_GENERATOR_SRC_KEYSPACE.defaultValue()); long sleepAfterElements = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(), UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue()); @@ -90,11 +93,27 @@ public static void main(String[] args) throws Exception { .addSource(new TtlStateUpdateSource(keySpace, sleepAfterElements, sleepTime)) .name("TtlStateUpdateSource") .keyBy(TtlStateUpdate::getKey) - .flatMap(new TtlVerifyUpdateFunction(ttlConfig, reportStatAfterUpdatesNum)) + .flatMap(new TtlVerifyUpdateFunction(ttlConfig, ttlTimeProvider, reportStatAfterUpdatesNum)) .name("TtlVerifyUpdateFunction") .addSink(new PrintSinkFunction<>()) .name("PrintFailedVerifications"); env.execute("State TTL test job"); } + + /** + * Sets the state backend to a new {@link StubStateBackend} which has a {@link MonotonicTTLTimeProvider}. + * + * @param env The {@link StreamExecutionEnvironment} of the job. + * @return The {@link MonotonicTTLTimeProvider}. + */ + private static MonotonicTTLTimeProvider setBackendWithCustomTTLTimeProvider(StreamExecutionEnvironment env) { + final MonotonicTTLTimeProvider ttlTimeProvider = new MonotonicTTLTimeProvider(); + + final StateBackend configuredBackend = env.getStateBackend(); + final StateBackend stubBackend = new StubStateBackend(configuredBackend, ttlTimeProvider); + env.setStateBackend(stubBackend); + + return ttlTimeProvider; + } } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/MonotonicTTLTimeProvider.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/MonotonicTTLTimeProvider.java new file mode 100644 index 0000000000000..0b5637d36c400 --- /dev/null +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/MonotonicTTLTimeProvider.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.runtime.state.ttl.TtlTimeProvider; + +import javax.annotation.concurrent.NotThreadSafe; + +import java.io.Serializable; + +/** + * A stub implementation of a {@link TtlTimeProvider} which guarantees that + * processing time increases monotonically. + */ +@NotThreadSafe +final class MonotonicTTLTimeProvider implements TtlTimeProvider, Serializable { + + private static final long serialVersionUID = 1L; + + /* + * The following variables are static because the whole TTLTimeProvider will go + * through serialization and, eventually, the state backend and the task executing + * the TtlVerifyUpdateFunction will have different instances of it. + * + * If these were not static, then the TtlVerifyUpdateFunction would e.g. freeze + * the time, but the backend would not be notified about it, resulting in inconsistent + * state. + * + * If the number of task slots per TM changes, then we may need to add also synchronization. + */ + + private static boolean timeIsFrozen = false; + + private static long lastReturnedProcessingTime = Long.MIN_VALUE; + + @Override + public long currentTimestamp() { + if (timeIsFrozen && lastReturnedProcessingTime != Long.MIN_VALUE) { + return lastReturnedProcessingTime; + } + + timeIsFrozen = true; + + final long currentProcessingTime = System.currentTimeMillis(); + if (currentProcessingTime < lastReturnedProcessingTime) { + return lastReturnedProcessingTime; + } + + lastReturnedProcessingTime = currentProcessingTime; + return lastReturnedProcessingTime; + } + + long unfreezeTime() { + timeIsFrozen = false; + return lastReturnedProcessingTime; + } +} diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java new file mode 100644 index 0000000000000..b93fa362d7fec --- /dev/null +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.query.TaskKvStateRegistry; +import org.apache.flink.runtime.state.AbstractKeyedStateBackend; +import org.apache.flink.runtime.state.CheckpointStorage; +import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation; +import org.apache.flink.runtime.state.KeyGroupRange; +import org.apache.flink.runtime.state.OperatorStateBackend; +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.ttl.TtlTimeProvider; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A stub implementation of the {@link StateBackend} that allows the use of + * a custom {@link TtlTimeProvider}. + */ +final class StubStateBackend implements StateBackend { + + private static final long serialVersionUID = 1L; + + private final TtlTimeProvider ttlTimeProvider; + + private final StateBackend backend; + + StubStateBackend(final StateBackend wrappedBackend, final TtlTimeProvider ttlTimeProvider) { + this.backend = checkNotNull(wrappedBackend); + this.ttlTimeProvider = checkNotNull(ttlTimeProvider); + } + + @Override + public CompletedCheckpointStorageLocation resolveCheckpoint(String externalPointer) throws IOException { + return backend.resolveCheckpoint(externalPointer); + } + + @Override + public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { + return backend.createCheckpointStorage(jobId); + } + + @Override + public AbstractKeyedStateBackend createKeyedStateBackend( + Environment env, + JobID jobID, + String operatorIdentifier, + TypeSerializer keySerializer, + int numberOfKeyGroups, + KeyGroupRange keyGroupRange, + TaskKvStateRegistry kvStateRegistry, + TtlTimeProvider ttlTimeProvider, + MetricGroup metricGroup) throws Exception { + + return backend.createKeyedStateBackend( + env, + jobID, + operatorIdentifier, + keySerializer, + numberOfKeyGroups, + keyGroupRange, + kvStateRegistry, + this.ttlTimeProvider, + metricGroup + ); + } + + @Override + public OperatorStateBackend createOperatorStateBackend(Environment env, String operatorIdentifier) throws Exception { + return backend.createOperatorStateBackend(env, operatorIdentifier); + } +} diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdate.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdate.java index e89b544cc669a..20e21ead233bf 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdate.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdate.java @@ -25,6 +25,9 @@ /** Randomly generated keyed state updates per state type. */ class TtlStateUpdate implements Serializable { + + private static final long serialVersionUID = 1L; + private final int key; @Nonnull diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdateSource.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdateSource.java index 6aff14e1cd3df..7404adf7771ff 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdateSource.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlStateUpdateSource.java @@ -33,6 +33,9 @@ * and waits for {@code sleepTime} to continue generation. */ class TtlStateUpdateSource extends RichParallelSourceFunction { + + private static final long serialVersionUID = 1L; + private final int maxKey; private final long sleepAfterElements; private final long sleepTime; diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlVerifyUpdateFunction.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlVerifyUpdateFunction.java index 3cfb0e2d86ec8..250041d0e34e4 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlVerifyUpdateFunction.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/TtlVerifyUpdateFunction.java @@ -33,7 +33,6 @@ import org.apache.flink.streaming.tests.verify.TtlVerificationContext; import org.apache.flink.streaming.tests.verify.ValueWithTs; import org.apache.flink.util.Collector; -import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,12 +40,14 @@ import javax.annotation.Nonnull; import java.io.Serializable; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + /** * Update state with TTL for each verifier. * @@ -62,21 +63,26 @@ * - verifies last update against previous updates * - emits verification context in case of failure */ -class TtlVerifyUpdateFunction - extends RichFlatMapFunction implements CheckpointedFunction { +class TtlVerifyUpdateFunction extends RichFlatMapFunction implements CheckpointedFunction { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(TtlVerifyUpdateFunction.class); @Nonnull private final StateTtlConfig ttlConfig; - private final long ttl; + private final MonotonicTTLTimeProvider ttlTimeProvider; private final UpdateStat stat; private transient Map states; private transient Map>> prevUpdatesByVerifierId; - TtlVerifyUpdateFunction(@Nonnull StateTtlConfig ttlConfig, long reportStatAfterUpdatesNum) { + TtlVerifyUpdateFunction( + @Nonnull StateTtlConfig ttlConfig, + MonotonicTTLTimeProvider ttlTimeProvider, + long reportStatAfterUpdatesNum) { this.ttlConfig = ttlConfig; - this.ttl = ttlConfig.getTtl().toMilliseconds(); + this.ttlTimeProvider = checkNotNull(ttlTimeProvider); this.stat = new UpdateStat(reportStatAfterUpdatesNum); } @@ -91,17 +97,13 @@ public void flatMap(TtlStateUpdate updates, Collector out) throws Except } private TtlVerificationContext generateUpdateAndVerificationContext( - TtlStateUpdate updates, TtlStateVerifier verifier) throws Exception { + TtlStateUpdate updates, + TtlStateVerifier verifier) throws Exception { + List> prevUpdates = getPrevUpdates(verifier.getId()); Object update = updates.getUpdate(verifier.getId()); TtlUpdateContext updateContext = performUpdate(verifier, update); - boolean clashes = updateClashesWithPrevUpdates(updateContext.getUpdateWithTs(), prevUpdates); - if (clashes) { - resetState(verifier.getId()); - prevUpdates = Collections.emptyList(); - updateContext = performUpdate(verifier, update); - } - stat.update(clashes, prevUpdates.size()); + stat.update(prevUpdates.size()); prevUpdatesByVerifierId.get(verifier.getId()).add(updateContext.getUpdateWithTs()); return new TtlVerificationContext<>(updates.getKey(), verifier.getId(), prevUpdates, updateContext); } @@ -113,33 +115,22 @@ private List> getPrevUpdates(String verifierId) throws Exception } private TtlUpdateContext performUpdate( - TtlStateVerifier verifier, Object update) throws Exception { + TtlStateVerifier verifier, + Object update) throws Exception { + + final long timestampBeforeUpdate = ttlTimeProvider.currentTimestamp(); State state = states.get(verifier.getId()); - long timestampBeforeUpdate = System.currentTimeMillis(); Object valueBeforeUpdate = verifier.get(state); verifier.update(state, update); Object updatedValue = verifier.get(state); - return new TtlUpdateContext<>(timestampBeforeUpdate, - valueBeforeUpdate, update, updatedValue, System.currentTimeMillis()); - } + final long timestampAfterUpdate = ttlTimeProvider.unfreezeTime(); - private boolean updateClashesWithPrevUpdates(ValueWithTs update, List> prevUpdates) { - return tooSlow(update) || - (!prevUpdates.isEmpty() && prevUpdates.stream().anyMatch(pu -> updatesClash(pu, update))); - } - - private boolean tooSlow(ValueWithTs update) { - return update.getTimestampAfterUpdate() - update.getTimestampBeforeUpdate() >= ttl; - } + checkState( + timestampAfterUpdate == timestampBeforeUpdate, + "Timestamps before and after the update do not match." + ); - private boolean updatesClash(ValueWithTs prevUpdate, ValueWithTs nextUpdate) { - return prevUpdate.getTimestampAfterUpdate() + ttl >= nextUpdate.getTimestampBeforeUpdate() && - prevUpdate.getTimestampBeforeUpdate() + ttl <= nextUpdate.getTimestampAfterUpdate(); - } - - private void resetState(String verifierId) { - states.get(verifierId).clear(); - prevUpdatesByVerifierId.get(verifierId).clear(); + return new TtlUpdateContext<>(valueBeforeUpdate, update, updatedValue, timestampAfterUpdate); } @Override @@ -153,7 +144,7 @@ public void initializeState(FunctionInitializationContext context) { .collect(Collectors.toMap(TtlStateVerifier::getId, v -> v.createState(context, ttlConfig))); prevUpdatesByVerifierId = TtlStateVerifier.VERIFIERS.stream() .collect(Collectors.toMap(TtlStateVerifier::getId, v -> { - Preconditions.checkNotNull(v); + checkNotNull(v); TypeSerializer> typeSerializer = new ValueWithTs.Serializer(v.getUpdateSerializer()); ListStateDescriptor> stateDesc = new ListStateDescriptor<>( "TtlPrevValueState_" + v.getId(), typeSerializer); @@ -165,22 +156,17 @@ public void initializeState(FunctionInitializationContext context) { private static class UpdateStat implements Serializable { final long reportStatAfterUpdatesNum; long updates = 0; - long clashes = 0; long prevUpdatesNum = 0; UpdateStat(long reportStatAfterUpdatesNum) { this.reportStatAfterUpdatesNum = reportStatAfterUpdatesNum; } - void update(boolean clash, long prevUpdatesSize) { + void update(long prevUpdatesSize) { updates++; - if (clash) { - clashes++; - } prevUpdatesNum += prevUpdatesSize; if (updates % reportStatAfterUpdatesNum == 0) { - LOG.info(String.format("Avg update chain length: %d, clash stat: %d/%d", - prevUpdatesNum / updates, clashes, updates)); + LOG.info(String.format("Avg update chain length: %d", prevUpdatesNum / updates)); } } } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/AbstractTtlStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/AbstractTtlStateVerifier.java index 7b6def24204ec..46becbbbc18c4 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/AbstractTtlStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/AbstractTtlStateVerifier.java @@ -86,15 +86,18 @@ public void update(@Nonnull State state, Object update) throws Exception { @Override public boolean verify(@Nonnull TtlVerificationContext verificationContextRaw) { TtlVerificationContext verificationContext = (TtlVerificationContext) verificationContextRaw; - List> updates = new ArrayList<>(verificationContext.getPrevUpdates()); - long currentTimestamp = verificationContext.getUpdateContext().getTimestampBeforeUpdate(); - GV prevValue = expected(updates, currentTimestamp); + long currentTimestamp = verificationContext.getUpdateContext().getTimestamp(); + GV valueBeforeUpdate = verificationContext.getUpdateContext().getValueBeforeUpdate(); + List> updates = new ArrayList<>(verificationContext.getPrevUpdates()); + GV expectedValueBeforeUpdate = expected(updates, currentTimestamp); + + GV valueAfterUpdate = verificationContext.getUpdateContext().getValueAfterUpdate(); ValueWithTs update = verificationContext.getUpdateContext().getUpdateWithTs(); - GV updatedValue = verificationContext.getUpdateContext().getUpdatedValue(); updates.add(update); - GV expectedValue = expected(updates, currentTimestamp); - return Objects.equals(valueBeforeUpdate, prevValue) && Objects.equals(updatedValue, expectedValue); + GV expectedValueAfterUpdate = expected(updates, currentTimestamp); + + return Objects.equals(valueBeforeUpdate, expectedValueBeforeUpdate) && Objects.equals(valueAfterUpdate, expectedValueAfterUpdate); } abstract GV expected(@Nonnull List> updates, long currentTimestamp); diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlAggregatingStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlAggregatingStateVerifier.java index 960bbe72401ff..8a629578679bd 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlAggregatingStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlAggregatingStateVerifier.java @@ -71,13 +71,13 @@ String expected(@Nonnull List> updates, long currentTimesta return null; } long acc = AGG_FUNC.createAccumulator(); - long lastTs = updates.get(0).getTimestampAfterUpdate(); + long lastTs = updates.get(0).getTimestamp(); for (ValueWithTs update : updates) { - if (expired(lastTs, update.getTimestampAfterUpdate())) { + if (expired(lastTs, update.getTimestamp())) { acc = AGG_FUNC.createAccumulator(); } acc = AGG_FUNC.add(update.getValue(), acc); - lastTs = update.getTimestampAfterUpdate(); + lastTs = update.getTimestamp(); } return expired(lastTs, currentTimestamp) ? null : AGG_FUNC.getResult(acc); } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlFoldingStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlFoldingStateVerifier.java index c1cc761b0f3f1..bcc8590586381 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlFoldingStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlFoldingStateVerifier.java @@ -74,13 +74,13 @@ Long expected(@Nonnull List> updates, long currentTimestamp return null; } long acc = INIT_VAL; - long lastTs = updates.get(0).getTimestampAfterUpdate(); + long lastTs = updates.get(0).getTimestamp(); for (ValueWithTs update : updates) { - if (expired(lastTs, update.getTimestampAfterUpdate())) { + if (expired(lastTs, update.getTimestamp())) { acc = INIT_VAL; } acc += update.getValue(); - lastTs = update.getTimestampAfterUpdate(); + lastTs = update.getTimestamp(); } return expired(lastTs, currentTimestamp) ? null : acc; } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlListStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlListStateVerifier.java index b355aa9861a48..4aed98ffd2fb0 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlListStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlListStateVerifier.java @@ -71,7 +71,7 @@ void updateInternal(@Nonnull ListState state, String update) throws Exce @Nonnull List expected(@Nonnull List> updates, long currentTimestamp) { return updates.stream() - .filter(u -> !expired(u.getTimestampAfterUpdate(), currentTimestamp)) + .filter(u -> !expired(u.getTimestamp(), currentTimestamp)) .map(ValueWithTs::getValue) .collect(Collectors.toList()); } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlMapStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlMapStateVerifier.java index a9d6b36f62f08..eeda78d73c213 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlMapStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlMapStateVerifier.java @@ -87,7 +87,7 @@ Map expected(@Nonnull List>> .collect(Collectors.groupingBy(u -> u.getValue().f0)) .entrySet().stream() .map(e -> e.getValue().get(e.getValue().size() - 1)) - .filter(u -> !expired(u.getTimestampAfterUpdate(), currentTimestamp)) + .filter(u -> !expired(u.getTimestamp(), currentTimestamp)) .map(ValueWithTs::getValue) .collect(Collectors.toMap(u -> u.f0, u -> u.f1)); } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlReducingStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlReducingStateVerifier.java index 773be05e04b5d..cd33ed08f4eb4 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlReducingStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlReducingStateVerifier.java @@ -73,13 +73,13 @@ Integer expected(@Nonnull List> updates, long currentTimest return null; } int acc = 0; - long lastTs = updates.get(0).getTimestampAfterUpdate(); + long lastTs = updates.get(0).getTimestamp(); for (ValueWithTs update : updates) { - if (expired(lastTs, update.getTimestampAfterUpdate())) { + if (expired(lastTs, update.getTimestamp())) { acc = 0; } acc += update.getValue(); - lastTs = update.getTimestampAfterUpdate(); + lastTs = update.getTimestamp(); } return expired(lastTs, currentTimestamp) ? null : acc; } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlUpdateContext.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlUpdateContext.java index 959340b408d9c..61bf9e22c6306 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlUpdateContext.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlUpdateContext.java @@ -24,25 +24,25 @@ /** Contains context relevant for state update with TTL. */ public class TtlUpdateContext implements Serializable { - private final long timestampBeforeUpdate; + private final GV valueBeforeUpdate; private final UV update; - private final GV updatedValue; - private final long timestampAfterUpdate; + private final GV valueAfterUpdate; + private final long timestamp; public TtlUpdateContext( - long timestampBeforeUpdate, - GV valueBeforeUpdate, UV update, GV updatedValue, - long timestampAfterUpdate) { + GV valueBeforeUpdate, + UV update, + GV updatedValue, + long timestamp) { this.valueBeforeUpdate = valueBeforeUpdate; this.update = update; - this.updatedValue = updatedValue; - this.timestampBeforeUpdate = timestampBeforeUpdate; - this.timestampAfterUpdate = timestampAfterUpdate; + this.valueAfterUpdate = updatedValue; + this.timestamp = timestamp; } - long getTimestampBeforeUpdate() { - return timestampBeforeUpdate; + long getTimestamp() { + return timestamp; } GV getValueBeforeUpdate() { @@ -51,21 +51,20 @@ GV getValueBeforeUpdate() { @Nonnull public ValueWithTs getUpdateWithTs() { - return new ValueWithTs<>(update, timestampBeforeUpdate, timestampAfterUpdate); + return new ValueWithTs<>(update, timestamp); } - GV getUpdatedValue() { - return updatedValue; + GV getValueAfterUpdate() { + return valueAfterUpdate; } @Override public String toString() { return "TtlUpdateContext{" + - "timestampBeforeUpdate=" + timestampBeforeUpdate + - ", valueBeforeUpdate=" + valueBeforeUpdate + + "valueBeforeUpdate=" + valueBeforeUpdate + ", update=" + update + - ", updatedValue=" + updatedValue + - ", timestampAfterUpdate=" + timestampAfterUpdate + + ", valueAfterUpdate=" + valueAfterUpdate + + ", timestamp=" + timestamp + '}'; } } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlValueStateVerifier.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlValueStateVerifier.java index fa4929b933f41..d8bdfd4e79de9 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlValueStateVerifier.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlValueStateVerifier.java @@ -61,6 +61,6 @@ String expected(@Nonnull List> updates, long currentTimestam return null; } ValueWithTs lastUpdate = updates.get(updates.size() - 1); - return expired(lastUpdate.getTimestampAfterUpdate(), currentTimestamp) ? null : lastUpdate.getValue(); + return expired(lastUpdate.getTimestamp(), currentTimestamp) ? null : lastUpdate.getValue(); } } diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlVerificationContext.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlVerificationContext.java index 4c985cd525b31..6d04c4cb21928 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlVerificationContext.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/TtlVerificationContext.java @@ -36,10 +36,10 @@ public class TtlVerificationContext implements Serializable { @SuppressWarnings("unchecked") public TtlVerificationContext( - int key, - @Nonnull String verifierId, - @Nonnull List> prevUpdates, - @Nonnull TtlUpdateContext updateContext) { + int key, + @Nonnull String verifierId, + @Nonnull List> prevUpdates, + @Nonnull TtlUpdateContext updateContext) { this.key = key; this.verifierId = verifierId; this.prevUpdates = new ArrayList<>(); diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/ValueWithTs.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/ValueWithTs.java index 9302377d9eddb..a4f30804e4fd2 100644 --- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/ValueWithTs.java +++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/verify/ValueWithTs.java @@ -30,33 +30,26 @@ /** User state value with timestamps before and after update. */ public class ValueWithTs implements Serializable { private final V value; - private final long timestampBeforeUpdate; - private final long timestampAfterUpdate; + private final long timestamp; - public ValueWithTs(V value, long timestampBeforeUpdate, long timestampAfterUpdate) { + public ValueWithTs(V value, long timestamp) { this.value = value; - this.timestampBeforeUpdate = timestampBeforeUpdate; - this.timestampAfterUpdate = timestampAfterUpdate; + this.timestamp = timestamp; } V getValue() { return value; } - public long getTimestampBeforeUpdate() { - return timestampBeforeUpdate; - } - - public long getTimestampAfterUpdate() { - return timestampAfterUpdate; + long getTimestamp() { + return timestamp; } @Override public String toString() { return "ValueWithTs{" + "value=" + value + - ", timestampBeforeUpdate=" + timestampBeforeUpdate + - ", timestampAfterUpdate=" + timestampAfterUpdate + + ", timestamp=" + timestamp + '}'; } @@ -64,7 +57,7 @@ public String toString() { public static class Serializer extends CompositeSerializer> { public Serializer(TypeSerializer userValueSerializer) { - super(true, userValueSerializer, LongSerializer.INSTANCE, LongSerializer.INSTANCE); + super(true, userValueSerializer, LongSerializer.INSTANCE); } @SuppressWarnings("unchecked") @@ -74,7 +67,7 @@ public Serializer(TypeSerializer userValueSerializer) { @Override public ValueWithTs createInstance(@Nonnull Object ... values) { - return new ValueWithTs<>(values[0], (Long) values[1], (Long) values[2]); + return new ValueWithTs<>(values[0], (Long) values[1]); } @Override @@ -88,9 +81,7 @@ protected Object getField(@Nonnull ValueWithTs value, int index) { case 0: return value.getValue(); case 1: - return value.getTimestampBeforeUpdate(); - case 2: - return value.getTimestampAfterUpdate(); + return value.getTimestamp(); default: throw new FlinkRuntimeException("Unexpected field index for ValueWithTs"); } @@ -99,8 +90,8 @@ protected Object getField(@Nonnull ValueWithTs value, int index) { @SuppressWarnings("unchecked") @Override protected CompositeSerializer> createSerializerInstance( - PrecomputedParameters precomputed, - TypeSerializer... originalSerializers) { + PrecomputedParameters precomputed, + TypeSerializer... originalSerializers) { return new Serializer(precomputed, (TypeSerializer) originalSerializers[0]); } } diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test-base/pom.xml b/flink-end-to-end-tests/flink-streaming-kafka-test-base/pom.xml new file mode 100644 index 0000000000000..71366483a41ef --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/pom.xml @@ -0,0 +1,44 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + 4.0.0 + + flink-streaming-kafka-test-base_${scala.binary.version} + flink-streaming-kafka-test-base + + jar + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + + diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/CustomWatermarkExtractor.java b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/CustomWatermarkExtractor.java new file mode 100644 index 0000000000000..8b306ac92972f --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/CustomWatermarkExtractor.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test.base; + +import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; +import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; +import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; +import org.apache.flink.streaming.api.watermark.Watermark; + +import javax.annotation.Nullable; + +/** + * A custom {@link AssignerWithPeriodicWatermarks}, that simply assumes that the input stream + * records are strictly ascending. + * + *

    Flink also ships some built-in convenience assigners, such as the + * {@link BoundedOutOfOrdernessTimestampExtractor} and {@link AscendingTimestampExtractor} + */ +public class CustomWatermarkExtractor implements AssignerWithPeriodicWatermarks { + + private static final long serialVersionUID = -742759155861320823L; + + private long currentTimestamp = Long.MIN_VALUE; + + @Override + public long extractTimestamp(KafkaEvent event, long previousElementTimestamp) { + // the inputs are assumed to be of format (message,timestamp) + this.currentTimestamp = event.getTimestamp(); + return event.getTimestamp(); + } + + @Nullable + @Override + public Watermark getCurrentWatermark() { + return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); + } +} diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEvent.java b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEvent.java similarity index 97% rename from flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEvent.java rename to flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEvent.java index a144fc38257a9..491228104e9b2 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEvent.java +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEvent.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.flink.streaming.examples.kafka; +package org.apache.flink.streaming.kafka.test.base; /** * The event type used in the {@link Kafka010Example}. diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEventSchema.java b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEventSchema.java similarity index 97% rename from flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEventSchema.java rename to flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEventSchema.java index 5b8e17dadca88..c2cc869e5d90d 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/KafkaEventSchema.java +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaEventSchema.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.flink.streaming.examples.kafka; +package org.apache.flink.streaming.kafka.test.base; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaExampleUtil.java b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaExampleUtil.java new file mode 100644 index 0000000000000..eab23e6fcfab9 --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/KafkaExampleUtil.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test.base; + +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +/** + * The util class for kafka example. + */ +public class KafkaExampleUtil { + + public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool) + throws Exception { + + if (parameterTool.getNumberOfParameters() < 5) { + System.out.println("Missing parameters!\n" + + "Usage: Kafka --input-topic --output-topic " + + "--bootstrap.servers " + + "--zookeeper.connect --group.id "); + throw new Exception("Missing parameters!\n" + + "Usage: Kafka --input-topic --output-topic " + + "--bootstrap.servers " + + "--zookeeper.connect --group.id "); + } + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.getConfig().disableSysoutLogging(); + env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)); + env.enableCheckpointing(5000); // create a checkpoint every 5 seconds + env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); + + return env; + } + +} diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/RollingAdditionMapper.java b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/RollingAdditionMapper.java new file mode 100644 index 0000000000000..927c3eacfaeb8 --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test-base/src/main/java/org/apache/flink/streaming/kafka/test/base/RollingAdditionMapper.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test.base; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.configuration.Configuration; + +/** + * A {@link RichMapFunction} that continuously outputs the current total frequency count of a key. + * The current total count is keyed state managed by Flink. + */ +public class RollingAdditionMapper extends RichMapFunction { + + private static final long serialVersionUID = 1180234853172462378L; + + private transient ValueState currentTotalCount; + + @Override + public KafkaEvent map(KafkaEvent event) throws Exception { + Integer totalCount = currentTotalCount.value(); + + if (totalCount == null) { + totalCount = 0; + } + totalCount += event.getFrequency(); + + currentTotalCount.update(totalCount); + + return new KafkaEvent(event.getWord(), totalCount, event.getTimestamp()); + } + + @Override + public void open(Configuration parameters) throws Exception { + currentTotalCount = getRuntimeContext().getState(new ValueStateDescriptor<>("currentTotalCount", Integer.class)); + } +} diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test/pom.xml b/flink-end-to-end-tests/flink-streaming-kafka-test/pom.xml new file mode 100644 index 0000000000000..a19881e5439ac --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test/pom.xml @@ -0,0 +1,84 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + 4.0.0 + + flink-streaming-kafka-test_${scala.binary.version} + flink-streaming-kafka-test + + + + org.apache.flink + flink-streaming-kafka-test-base_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-connector-kafka_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + fat-jar-kafka-example + package + + shade + + + false + false + false + + + org.apache.flink.streaming.kafka.test.KafkaExample + + + KafkaExample + + + + + + + + diff --git a/flink-end-to-end-tests/flink-streaming-kafka-test/src/main/java/org/apache/flink/streaming/kafka/test/KafkaExample.java b/flink-end-to-end-tests/flink-streaming-kafka-test/src/main/java/org/apache/flink/streaming/kafka/test/KafkaExample.java new file mode 100644 index 0000000000000..70885402b2abd --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka-test/src/main/java/org/apache/flink/streaming/kafka/test/KafkaExample.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test; + +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor; +import org.apache.flink.streaming.kafka.test.base.KafkaEvent; +import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema; +import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil; +import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper; +import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper; + +/** + * A simple example that shows how to read from and write to modern Kafka. This will read String messages + * from the input topic, parse them into a POJO type {@link KafkaEvent}, group by some key, and finally + * perform a rolling addition on each key for which the results are written back to another topic. + * + *

    This example also demonstrates using a watermark assigner to generate per-partition + * watermarks directly in the Flink Kafka consumer. For demonstration purposes, it is assumed that + * the String messages are of formatted as a (word,frequency,timestamp) tuple. + * + *

    Example usage: + * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 + * --zookeeper.connect localhost:2181 --group.id myconsumer + */ +public class KafkaExample extends KafkaExampleUtil { + + public static void main(String[] args) throws Exception { + // parse input arguments + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool); + + DataStream input = env + .addSource( + new FlinkKafkaConsumer<>( + parameterTool.getRequired("input-topic"), + new KafkaEventSchema(), + parameterTool.getProperties()) + .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) + .keyBy("word") + .map(new RollingAdditionMapper()); + + input.addSink( + new FlinkKafkaProducer<>( + parameterTool.getRequired("output-topic"), + new KeyedSerializationSchemaWrapper<>(new KafkaEventSchema()), + parameterTool.getProperties(), + FlinkKafkaProducer.Semantic.EXACTLY_ONCE)); + + env.execute("Modern Kafka Example"); + } + +} diff --git a/flink-end-to-end-tests/flink-streaming-kafka010-test/pom.xml b/flink-end-to-end-tests/flink-streaming-kafka010-test/pom.xml new file mode 100644 index 0000000000000..53e1abaf14dd3 --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka010-test/pom.xml @@ -0,0 +1,84 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + 4.0.0 + + flink-streaming-kafka010-test_${scala.binary.version} + flink-streaming-kafka010-test + + + + org.apache.flink + flink-streaming-kafka-test-base_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-connector-kafka-0.10_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + fat-jar-kafka-example + package + + shade + + + false + false + false + + + org.apache.flink.streaming.kafka.test.Kafka010Example + + + Kafka010Example + + + + + + + + diff --git a/flink-end-to-end-tests/flink-streaming-kafka010-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka010Example.java b/flink-end-to-end-tests/flink-streaming-kafka010-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka010Example.java new file mode 100644 index 0000000000000..0b97179425861 --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka010-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka010Example.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test; + +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; +import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor; +import org.apache.flink.streaming.kafka.test.base.KafkaEvent; +import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema; +import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil; +import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper; + +/** + * A simple example that shows how to read from and write to Kafka. This will read String messages + * from the input topic, parse them into a POJO type {@link KafkaEvent}, group by some key, and finally + * perform a rolling addition on each key for which the results are written back to another topic. + * + *

    This example also demonstrates using a watermark assigner to generate per-partition + * watermarks directly in the Flink Kafka consumer. For demonstration purposes, it is assumed that + * the String messages are of formatted as a (word,frequency,timestamp) tuple. + * + *

    Example usage: + * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer + */ +public class Kafka010Example { + + public static void main(String[] args) throws Exception { + // parse input arguments + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool); + + DataStream input = env + .addSource( + new FlinkKafkaConsumer010<>( + parameterTool.getRequired("input-topic"), + new KafkaEventSchema(), + parameterTool.getProperties()) + .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) + .keyBy("word") + .map(new RollingAdditionMapper()); + + input.addSink( + new FlinkKafkaProducer010<>( + parameterTool.getRequired("output-topic"), + new KafkaEventSchema(), + parameterTool.getProperties())); + + env.execute("Kafka 0.10 Example"); + } + +} diff --git a/flink-end-to-end-tests/flink-streaming-kafka011-test/pom.xml b/flink-end-to-end-tests/flink-streaming-kafka011-test/pom.xml new file mode 100644 index 0000000000000..577d3a4b2c35b --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka011-test/pom.xml @@ -0,0 +1,85 @@ + + + + + flink-end-to-end-tests + org.apache.flink + 1.7-SNAPSHOT + .. + + 4.0.0 + + flink-streaming-kafka011-test_${scala.binary.version} + flink-streaming-kafka011-test + jar + + + + org.apache.flink + flink-streaming-kafka-test-base_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-connector-kafka-0.11_${scala.binary.version} + ${project.version} + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + fat-jar-kafka-example + package + + shade + + + false + false + false + + + org.apache.flink.streaming.kafka.test.Kafka011Example + + + Kafka011Example + + + + + + + + diff --git a/flink-end-to-end-tests/flink-streaming-kafka011-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka011Example.java b/flink-end-to-end-tests/flink-streaming-kafka011-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka011Example.java new file mode 100644 index 0000000000000..1f877c5a5b840 --- /dev/null +++ b/flink-end-to-end-tests/flink-streaming-kafka011-test/src/main/java/org/apache/flink/streaming/kafka/test/Kafka011Example.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.kafka.test; + +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; +import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor; +import org.apache.flink.streaming.kafka.test.base.KafkaEvent; +import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema; +import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil; +import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper; + +/** + * A simple example that shows how to read from and write to Kafka. This will read String messages + * from the input topic, parse them into a POJO type {@link KafkaEvent}, group by some key, and finally + * perform a rolling addition on each key for which the results are written back to another topic. + * + *

    This example also demonstrates using a watermark assigner to generate per-partition + * watermarks directly in the Flink Kafka consumer. For demonstration purposes, it is assumed that + * the String messages are of formatted as a (word,frequency,timestamp) tuple. + * + *

    Example usage: + * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer + */ +public class Kafka011Example { + + public static void main(String[] args) throws Exception { + // parse input arguments + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool); + + DataStream input = env + .addSource( + new FlinkKafkaConsumer011<>( + parameterTool.getRequired("input-topic"), + new KafkaEventSchema(), + parameterTool.getProperties()) + .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) + .keyBy("word") + .map(new RollingAdditionMapper()); + + input.addSink( + new FlinkKafkaProducer011<>( + parameterTool.getRequired("output-topic"), + new KafkaEventSchema(), + parameterTool.getProperties())); + + env.execute("Kafka 0.11 Example"); + } + +} diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index 6b31881f684d6..c7e010e4e10e0 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -35,6 +35,7 @@ under the License. flink-end-to-end-tests + flink-cli-test flink-parent-child-classloading-test flink-dataset-allround-test flink-datastream-allround-test @@ -55,6 +56,15 @@ under the License. flink-sql-client-test flink-streaming-file-sink-test flink-state-evolution-test + flink-e2e-test-utils + flink-end-to-end-tests-common + flink-metrics-availability-test + flink-metrics-reporter-prometheus-test + flink-heavy-deployment-stress-test + flink-streaming-kafka-test-base + flink-streaming-kafka-test + flink-streaming-kafka011-test + flink-streaming-kafka010-test @@ -66,6 +76,22 @@ under the License. true + + org.apache.maven.plugins + maven-surefire-plugin + + + + default-test + none + + + integration-tests + none + + + diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index 6a6c8b6729b4a..24d77c3841fcc 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -49,22 +49,9 @@ echo "Flink distribution directory: $FLINK_DIR" # those checks are disabled, one should take care that a proper checks are performed in the tests itself that ensure that the test finished # in an expected state. -run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25" - -run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb" -run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" "skip_check_exceptions" - -run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh" "skip_check_exceptions" - -run_test "Running HA (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file true false" "skip_check_exceptions" -run_test "Running HA (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file false false" "skip_check_exceptions" -run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false" "skip_check_exceptions" -run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true" "skip_check_exceptions" - -run_test "Running HA per-job cluster (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file true false" "skip_check_exceptions" -run_test "Running HA per-job cluster (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file false false" "skip_check_exceptions" -run_test "Running HA per-job cluster (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true false" "skip_check_exceptions" -run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true true" "skip_check_exceptions" +################################################################################ +# Checkpointing tests +################################################################################ run_test "Resuming Savepoint (file, async, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file true" run_test "Resuming Savepoint (file, sync, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file false" @@ -97,19 +84,50 @@ run_test "Resuming Externalized Checkpoint after terminal failure (file, sync) e run_test "Resuming Externalized Checkpoint after terminal failure (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true false true" run_test "Resuming Externalized Checkpoint after terminal failure (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true true true" +################################################################################ +# Docker +################################################################################ + +run_test "Running Kerberized YARN on Docker test " "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" + +################################################################################ +# High Availability +################################################################################ + +run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh" "skip_check_exceptions" + +run_test "Running HA (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file true false" "skip_check_exceptions" +run_test "Running HA (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file false false" "skip_check_exceptions" +run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false" "skip_check_exceptions" +run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true" "skip_check_exceptions" + +run_test "Running HA per-job cluster (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file true false" "skip_check_exceptions" +run_test "Running HA per-job cluster (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file false false" "skip_check_exceptions" +run_test "Running HA per-job cluster (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true false" "skip_check_exceptions" +run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true true" "skip_check_exceptions" + +################################################################################ +# Kubernetes +################################################################################ + +run_test "Run Kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh" + +################################################################################ +# Miscellaneous +################################################################################ + +run_test "Flink CLI end-to-end test" "$END_TO_END_DIR/test-scripts/test_cli.sh" + +run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb" +run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" "skip_check_exceptions" + run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_allround.sh" run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh" "skip_check_exceptions" run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh" "skip_check_exceptions" run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh" "skip_check_exceptions" +run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh s3" "skip_check_exceptions" run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false false" "skip_check_exceptions" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false true" "skip_check_exceptions" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false" "skip_check_exceptions" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true false" "skip_check_exceptions" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false true" "skip_check_exceptions" -run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true" "skip_check_exceptions" - run_test "Elasticsearch (v1.7.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1 https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz" run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz" run_test "Elasticsearch (v5.1.2) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz" @@ -123,9 +141,25 @@ run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_D run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" -run_test "Running Kerberized YARN on Docker test " "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" - run_test "SQL Client end-to-end test" "$END_TO_END_DIR/test-scripts/test_sql_client.sh" +run_test "SQL Client end-to-end test for Kafka 0.10" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka010.sh" +run_test "SQL Client end-to-end test for Kafka 0.11" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka011.sh" +run_test "SQL Client end-to-end test for modern Kafka" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka.sh" + +run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_heavy_deployment.sh" "skip_check_exceptions" + +run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25" + +################################################################################ +# Sticky Scheduling +################################################################################ + +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false true" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false true" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true" "skip_check_exceptions" printf "\n[PASS] All tests passed\n" exit 0 diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh index 15b9b152f7740..c9fab30491645 100755 --- a/flink-end-to-end-tests/run-pre-commit-tests.sh +++ b/flink-end-to-end-tests/run-pre-commit-tests.sh @@ -53,7 +53,9 @@ run_test "State Evolution end-to-end test" "$END_TO_END_DIR/test-scripts/test_st run_test "Batch Python Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_python_wordcount.sh" run_test "Streaming Python Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_python_wordcount.sh" run_test "Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh" -run_test "Kafka end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh" +run_test "Kafka 0.10 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh" +run_test "Kafka 0.11 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka011.sh" +run_test "Modern Kafka end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka.sh" run_test "class loading end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_classloader.sh" run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_hadoop_s3a.sh" run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh" diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index 09c42af8742ea..2f4c849960026 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -17,6 +17,8 @@ # limitations under the License. ################################################################################ +# Enable this line when developing a new end-to-end test +#set -Eexuo pipefail set -o pipefail if [[ -z $FLINK_DIR ]]; then @@ -39,13 +41,20 @@ echo "Flink dist directory: $FLINK_DIR" FLINK_VERSION=$(cat ${END_TO_END_DIR}/pom.xml | sed -n 's/.*\(.*\)<\/version>/\1/p') -USE_SSL=OFF # set via set_conf_ssl(), reset via revert_default_config() TEST_ROOT=`pwd -P` TEST_INFRA_DIR="$END_TO_END_DIR/test-scripts/" cd $TEST_INFRA_DIR TEST_INFRA_DIR=`pwd -P` cd $TEST_ROOT +NODENAME=`hostname -f` + +# REST_PROTOCOL and CURL_SSL_ARGS can be modified in common_ssl.sh if SSL is activated +# they should be used in curl command to query Flink REST API +REST_PROTOCOL="http" +CURL_SSL_ARGS="" +source "${TEST_INFRA_DIR}/common_ssl.sh" + function print_mem_use_osx { declare -a mem_types=("active" "inactive" "wired down") used="" @@ -85,7 +94,8 @@ function revert_default_config() { mv -f $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml fi - USE_SSL=OFF + REST_PROTOCOL="http" + CURL_SSL_ARGS="" } function set_conf() { @@ -168,46 +178,6 @@ function get_node_ip { echo ${ip_addr} } -function set_conf_ssl { - - # clean up the dir that will be used for SSL certificates and trust stores - if [ -e "${TEST_DATA_DIR}/ssl" ]; then - echo "File ${TEST_DATA_DIR}/ssl exists. Deleting it..." - rm -rf "${TEST_DATA_DIR}/ssl" - fi - mkdir -p "${TEST_DATA_DIR}/ssl" - - NODENAME=`hostname -f` - SANSTRING="dns:${NODENAME}" - for NODEIP in $(get_node_ip) ; do - SANSTRING="${SANSTRING},ip:${NODEIP}" - done - - echo "Using SAN ${SANSTRING}" - - # create certificates - keytool -genkeypair -alias ca -keystore "${TEST_DATA_DIR}/ssl/ca.keystore" -dname "CN=Sample CA" -storepass password -keypass password -keyalg RSA -ext bc=ca:true - keytool -keystore "${TEST_DATA_DIR}/ssl/ca.keystore" -storepass password -alias ca -exportcert > "${TEST_DATA_DIR}/ssl/ca.cer" - keytool -importcert -keystore "${TEST_DATA_DIR}/ssl/ca.truststore" -alias ca -storepass password -noprompt -file "${TEST_DATA_DIR}/ssl/ca.cer" - - keytool -genkeypair -alias node -keystore "${TEST_DATA_DIR}/ssl/node.keystore" -dname "CN=${NODENAME}" -ext SAN=${SANSTRING} -storepass password -keypass password -keyalg RSA - keytool -certreq -keystore "${TEST_DATA_DIR}/ssl/node.keystore" -storepass password -alias node -file "${TEST_DATA_DIR}/ssl/node.csr" - keytool -gencert -keystore "${TEST_DATA_DIR}/ssl/ca.keystore" -storepass password -alias ca -ext SAN=${SANSTRING} -infile "${TEST_DATA_DIR}/ssl/node.csr" -outfile "${TEST_DATA_DIR}/ssl/node.cer" - keytool -importcert -keystore "${TEST_DATA_DIR}/ssl/node.keystore" -storepass password -file "${TEST_DATA_DIR}/ssl/ca.cer" -alias ca -noprompt - keytool -importcert -keystore "${TEST_DATA_DIR}/ssl/node.keystore" -storepass password -file "${TEST_DATA_DIR}/ssl/node.cer" -alias node -noprompt - - # adapt config - # (here we rely on security.ssl.enabled enabling SSL for all components and internal as well as - # external communication channels) - set_conf security.ssl.enabled true - set_conf security.ssl.keystore ${TEST_DATA_DIR}/ssl/node.keystore - set_conf security.ssl.keystore-password password - set_conf security.ssl.key-password password - set_conf security.ssl.truststore ${TEST_DATA_DIR}/ssl/ca.truststore - set_conf security.ssl.truststore-password password - USE_SSL=ON -} - function start_ha_cluster { create_ha_config start_local_zk @@ -241,26 +211,23 @@ function start_local_zk { function wait_dispatcher_running { # wait at most 10 seconds until the dispatcher is up - local QUERY_URL - if [ "x$USE_SSL" = "xON" ]; then - QUERY_URL="http://localhost:8081/taskmanagers" - else - QUERY_URL="https://localhost:8081/taskmanagers" - fi - for i in {1..10}; do + local QUERY_URL="${REST_PROTOCOL}://${NODENAME}:8081/taskmanagers" + local TIMEOUT=20 + for i in $(seq 1 ${TIMEOUT}); do # without the || true this would exit our script if the JobManager is not yet up - QUERY_RESULT=$(curl "$QUERY_URL" 2> /dev/null || true) + QUERY_RESULT=$(curl ${CURL_SSL_ARGS} "$QUERY_URL" 2> /dev/null || true) # ensure the taskmanagers field is there at all and is not empty if [[ ${QUERY_RESULT} =~ \{\"taskmanagers\":\[.+\]\} ]]; then - echo "Dispatcher REST endpoint is up." - break + return fi echo "Waiting for dispatcher REST endpoint to come up..." sleep 1 done + echo "Dispatcher REST endpoint has not started within a timeout of ${TIMEOUT} sec" + exit 1 } function start_cluster { @@ -278,32 +245,49 @@ function start_taskmanagers { } function start_and_wait_for_tm { - - tm_query_result=$(curl -s "http://localhost:8081/taskmanagers") - + tm_query_result=`query_running_tms` # we assume that the cluster is running if ! [[ ${tm_query_result} =~ \{\"taskmanagers\":\[.*\]\} ]]; then echo "Your cluster seems to be unresponsive at the moment: ${tm_query_result}" 1>&2 exit 1 fi - running_tms=`curl -s "http://localhost:8081/taskmanagers" | grep -o "id" | wc -l` - + running_tms=`query_number_of_running_tms` ${FLINK_DIR}/bin/taskmanager.sh start + wait_for_number_of_running_tms $((running_tms+1)) +} + +function query_running_tms { + local url="${REST_PROTOCOL}://${NODENAME}:8081/taskmanagers" + curl ${CURL_SSL_ARGS} -s "${url}" +} + +function query_number_of_running_tms { + query_running_tms | grep -o "id" | wc -l +} - for i in {1..10}; do - local new_running_tms=`curl -s "http://localhost:8081/taskmanagers" | grep -o "id" | wc -l` - if [ $((new_running_tms-running_tms)) -eq 0 ]; then - echo "TaskManager is not yet up." +function wait_for_number_of_running_tms { + local TM_NUM_TO_WAIT=${1} + local TIMEOUT_COUNTER=10 + local TIMEOUT_INC=4 + local TIMEOUT=$(( $TIMEOUT_COUNTER * $TIMEOUT_INC )) + local TM_NUM_TEXT="Number of running task managers" + for i in $(seq 1 ${TIMEOUT_COUNTER}); do + local TM_NUM=`query_number_of_running_tms` + if [ $((TM_NUM - TM_NUM_TO_WAIT)) -eq 0 ]; then + echo "${TM_NUM_TEXT} has reached ${TM_NUM_TO_WAIT}." + return else - echo "TaskManager is up." - break + echo "${TM_NUM_TEXT} ${TM_NUM} is not yet ${TM_NUM_TO_WAIT}." fi - sleep 4 + sleep ${TIMEOUT_INC} done + echo "${TM_NUM_TEXT} has not reached ${TM_NUM_TO_WAIT} within a timeout of ${TIMEOUT} sec" + exit 1 } function check_logs_for_errors { + echo "Checking for errors..." error_count=$(grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \ | grep -v "RetriableCommitFailedException" \ | grep -v "NoAvailableBrokersException" \ @@ -312,7 +296,7 @@ function check_logs_for_errors { | grep -v "AskTimeoutException" \ | grep -v "Error while loading kafka-version.properties" \ | grep -v "WARN akka.remote.transport.netty.NettyTransport" \ - | grep -v "WARN org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline" \ + | grep -v "WARN org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline" \ | grep -v "jvm-exit-on-fatal-error" \ | grep -v '^INFO:.*AWSErrorCode=\[400 Bad Request\].*ServiceEndpoint=\[https://.*\.s3\.amazonaws\.com\].*RequestType=\[HeadBucketRequest\]' \ | grep -v "RejectedExecutionException" \ @@ -320,15 +304,20 @@ function check_logs_for_errors { | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException" \ | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration" \ | grep -v "org.apache.flink.fs.shaded.hadoop3.org.apache.commons.beanutils.FluentPropertyBeanIntrospector - Error when creating PropertyDescriptor for public final void org.apache.flink.fs.shaded.hadoop3.org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property." \ - | grep -ic "error") + | grep -v "Error while loading kafka-version.properties :null" \ + | grep -v "Failed Elasticsearch item request" \ + | grep -ic "error" || true) if [[ ${error_count} -gt 0 ]]; then echo "Found error in log files:" cat $FLINK_DIR/log/* EXIT_CODE=1 + else + echo "No errors in log files." fi } function check_logs_for_exceptions { + echo "Checking for exceptions..." exception_count=$(grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \ | grep -v "RetriableCommitFailedException" \ | grep -v "NoAvailableBrokersException" \ @@ -347,19 +336,28 @@ function check_logs_for_exceptions { | grep -v "java.lang.Exception: Execution was suspended" \ | grep -v "java.io.InvalidClassException: org.apache.flink.formats.avro.typeutils.AvroSerializer" \ | grep -v "Caused by: java.lang.Exception: JobManager is shutting down" \ - | grep -ic "exception") + | grep -v "java.lang.Exception: Artificial failure" \ + | grep -v "org.apache.flink.runtime.checkpoint.decline" \ + | grep -v "org.elasticsearch.ElasticsearchException" \ + | grep -v "Elasticsearch exception" \ + | grep -ic "exception" || true) if [[ ${exception_count} -gt 0 ]]; then echo "Found exception in log files:" cat $FLINK_DIR/log/* EXIT_CODE=1 + else + echo "No exceptions in log files." fi } function check_logs_for_non_empty_out_files { + echo "Checking for non-empty .out files..." if grep -ri "." $FLINK_DIR/log/*.out > /dev/null; then echo "Found non-empty .out files:" cat $FLINK_DIR/log/*.out EXIT_CODE=1 + else + echo "No non-empty .out files." fi } @@ -373,7 +371,9 @@ function stop_cluster { "$FLINK_DIR"/bin/stop-cluster.sh # stop zookeeper only if there are processes running - if ! [ "`jps | grep 'FlinkZooKeeperQuorumPeer' | wc -l`" = "0" ]; then + zookeeper_process_count=$(jps | grep -c 'FlinkZooKeeperQuorumPeer' || true) + if [[ ${zookeeper_process_count} -gt 0 ]]; then + echo "Stopping zookeeper..." "$FLINK_DIR"/bin/zookeeper.sh stop fi } @@ -397,17 +397,20 @@ function wait_for_job_state_transition { } function wait_job_running { - for i in {1..10}; do + local TIMEOUT=10 + for i in $(seq 1 ${TIMEOUT}); do JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1") if [[ "$JOB_LIST_RESULT" == "" ]]; then echo "Job ($1) is not yet running." else echo "Job ($1) is running." - break + return fi sleep 1 done + echo "Job ($1) has not started within a timeout of ${TIMEOUT} sec" + exit 1 } function wait_job_terminal_state { @@ -417,7 +420,7 @@ function wait_job_terminal_state { echo "Waiting for job ($job) to reach terminal state $terminal_state ..." while : ; do - N=$(grep -o "Job $job reached globally terminal state $terminal_state" $FLINK_DIR/log/*standalonesession*.log | tail -1) + N=$(grep -o "Job $job reached globally terminal state $terminal_state" $FLINK_DIR/log/*standalonesession*.log | tail -1 || true) if [[ -z $N ]]; then sleep 1 @@ -436,6 +439,16 @@ function cancel_job { } function check_result_hash { + local error_code=0 + check_result_hash_no_exit "$@" || error_code=$? + + if [ "$error_code" != "0" ] + then + exit $error_code + fi +} + +function check_result_hash_no_exit { local name=$1 local outfile_prefix=$2 local expected=$3 @@ -447,55 +460,19 @@ function check_result_hash { actual=$(LC_ALL=C sort $outfile_prefix* | md5sum | awk '{print $1}') else echo "Neither 'md5' nor 'md5sum' binary available." - exit 2 + return 2 fi if [[ "$actual" != "$expected" ]] then echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected." echo "head hexdump of actual:" head $outfile_prefix* | hexdump -c - exit 1 + return 1 else echo "pass $name" # Output files are left behind in /tmp fi -} - -function s3_put { - local_file=$1 - bucket=$2 - s3_file=$3 - resource="/${bucket}/${s3_file}" - contentType="application/octet-stream" - dateValue=`date -R` - stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}" - s3Key=$ARTIFACTS_AWS_ACCESS_KEY - s3Secret=$ARTIFACTS_AWS_SECRET_KEY - signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64` - curl -X PUT -T "${local_file}" \ - -H "Host: ${bucket}.s3.amazonaws.com" \ - -H "Date: ${dateValue}" \ - -H "Content-Type: ${contentType}" \ - -H "Authorization: AWS ${s3Key}:${signature}" \ - https://${bucket}.s3.amazonaws.com/${s3_file} -} - -function s3_delete { - bucket=$1 - s3_file=$2 - resource="/${bucket}/${s3_file}" - contentType="application/octet-stream" - dateValue=`date -R` - stringToSign="DELETE\n\n${contentType}\n${dateValue}\n${resource}" - s3Key=$ARTIFACTS_AWS_ACCESS_KEY - s3Secret=$ARTIFACTS_AWS_SECRET_KEY - signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64` - curl -X DELETE \ - -H "Host: ${bucket}.s3.amazonaws.com" \ - -H "Date: ${dateValue}" \ - -H "Content-Type: ${contentType}" \ - -H "Authorization: AWS ${s3Key}:${signature}" \ - https://${bucket}.s3.amazonaws.com/${s3_file} + return 0 } # This function starts the given number of task managers and monitors their processes. @@ -525,9 +502,9 @@ function tm_kill_all { # Kills all processes that match the given name. function kill_all { - local pid=`jps | grep -E "${1}" | cut -d " " -f 1` - kill ${pid} 2> /dev/null - wait ${pid} 2> /dev/null + local pid=`jps | grep -E "${1}" | cut -d " " -f 1 || true` + kill ${pid} 2> /dev/null || true + wait ${pid} 2> /dev/null || true } function kill_random_taskmanager { @@ -551,7 +528,7 @@ function get_job_metric { local job_id=$1 local metric_name=$2 - local json=$(curl -s http://localhost:8081/jobs/${job_id}/metrics?get=${metric_name}) + local json=$(curl ${CURL_SSL_ARGS} -s ${REST_PROTOCOL}://${NODENAME}:8081/jobs/${job_id}/metrics?get=${metric_name}) local metric_value=$(echo ${json} | sed -n 's/.*"value":"\(.*\)".*/\1/p') echo ${metric_value} @@ -667,11 +644,6 @@ function clean_stdout_files { echo "Deleted all stdout files under ${FLINK_DIR}/log/" } -function clean_log_files { - rm ${FLINK_DIR}/log/* - echo "Deleted all files under ${FLINK_DIR}/log/" -} - # Expect a string to appear in the log files of the task manager before a given timeout # $1: expected string # $2: timeout in seconds @@ -691,3 +663,46 @@ function expect_in_taskmanager_logs { fi done } + +function wait_for_restart_to_complete { + local base_num_restarts=$1 + local jobid=$2 + + local current_num_restarts=${base_num_restarts} + local expected_num_restarts=$((current_num_restarts + 1)) + + echo "Waiting for restart to happen" + while ! [[ ${current_num_restarts} -eq ${expected_num_restarts} ]]; do + sleep 5 + current_num_restarts=$(get_job_metric ${jobid} "fullRestarts") + if [[ -z ${current_num_restarts} ]]; then + current_num_restarts=${base_num_restarts} + fi + done +} + +function find_latest_completed_checkpoint { + local checkpoint_root_directory=$1 + # a completed checkpoint must contain the _metadata file + local checkpoint_meta_file=$(ls -d ${checkpoint_root_directory}/chk-[1-9]*/_metadata | sort -Vr | head -n1) + echo "$(dirname "${checkpoint_meta_file}")" +} + +function retry_times() { + local retriesNumber=$1 + local backoff=$2 + local command=${@:3} + + for (( i = 0; i < ${retriesNumber}; i++ )) + do + if ${command}; then + return 0 + fi + + echo "Command: ${command} failed. Retrying..." + sleep ${backoff} + done + + echo "Command: ${command} failed ${retriesNumber} times." + return 1 +} diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh new file mode 100644 index 0000000000000..63fa941387a8c --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/common_s3.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +if [[ -z "$IT_CASE_S3_BUCKET" ]]; then + echo "Did not find AWS environment variables, NOT running the e2e test." + exit 0 +else + echo "Found AWS bucket $IT_CASE_S3_BUCKET, running the e2e test." +fi + +if [[ -z "$IT_CASE_S3_ACCESS_KEY" ]]; then + echo "Did not find AWS environment variables, NOT running the e2e test." + exit 0 +else + echo "Found AWS access key, running the e2e test." +fi + +if [[ -z "$IT_CASE_S3_SECRET_KEY" ]]; then + echo "Did not find AWS environment variables, NOT running the e2e test." + exit 0 +else + echo "Found AWS secret key, running the e2e test." +fi + +# export credentials into environment variables for AWS client +export AWS_REGION="${AWS_REGION:-us-east-1}" +export AWS_ACCESS_KEY_ID="$IT_CASE_S3_ACCESS_KEY" +export AWS_SECRET_ACCESS_KEY="$IT_CASE_S3_SECRET_KEY" + +AWS_ACCESS_KEY=$IT_CASE_S3_ACCESS_KEY +AWS_SECRET_KEY=$IT_CASE_S3_SECRET_KEY + +S3_TEST_DATA_WORDS_URI="s3://$IT_CASE_S3_BUCKET/static/words" + +s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.jar" + +################################### +# Setup Flink s3 access. +# +# Globals: +# FLINK_DIR +# IT_CASE_S3_ACCESS_KEY +# IT_CASE_S3_SECRET_KEY +# Arguments: +# $1 - s3 filesystem type (hadoop/presto) +# Returns: +# None +################################### +function s3_setup { + # make sure we delete the file at the end + function s3_cleanup { + rm $FLINK_DIR/lib/flink-s3-fs*.jar + + # remove any leftover settings + sed -i -e 's/s3.access-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" + sed -i -e 's/s3.secret-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" + } + trap s3_cleanup EXIT + + cp $FLINK_DIR/opt/flink-s3-fs-$1-*.jar $FLINK_DIR/lib/ + echo "s3.access-key: $IT_CASE_S3_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" + echo "s3.secret-key: $IT_CASE_S3_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" +} + +################################### +# List s3 objects by full path prefix. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - s3 full path key prefix +# Returns: +# List of s3 object keys, separated by newline +################################### +function s3_list { + AWS_REGION=$AWS_REGION \ + ${s3util} --action listByFullPathPrefix --s3prefix "$1" --bucket $IT_CASE_S3_BUCKET +} + +################################### +# Download s3 object. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - local path to save file +# $2 - s3 object key +# Returns: +# None +################################### +function s3_get { + AWS_REGION=$AWS_REGION \ + ${s3util} --action downloadFile --localFile "$1" --s3file "$2" --bucket $IT_CASE_S3_BUCKET +} + +################################### +# Download s3 objects to folder by full path prefix. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - local path to save folder with files +# $2 - s3 key full path prefix +# $3 - s3 file name prefix w/o directory to filter files by name (optional) +# Returns: +# None +################################### +function s3_get_by_full_path_and_filename_prefix { + local file_prefix="${3-}" + AWS_REGION=$AWS_REGION \ + ${s3util} --action downloadByFullPathAndFileNamePrefix \ + --localFolder "$1" --s3prefix "$2" --s3filePrefix "${file_prefix}" --bucket $IT_CASE_S3_BUCKET +} + +################################### +# Upload file to s3 object. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - local file to upload +# $2 - s3 bucket +# $3 - s3 object key +# Returns: +# None +################################### +function s3_put { + local_file=$1 + bucket=$2 + s3_file=$3 + resource="/${bucket}/${s3_file}" + contentType="application/octet-stream" + dateValue=`date -R` + stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}" + s3Key=$IT_CASE_S3_ACCESS_KEY + s3Secret=$IT_CASE_S3_SECRET_KEY + signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64` + curl -X PUT -T "${local_file}" \ + -H "Host: ${bucket}.s3.amazonaws.com" \ + -H "Date: ${dateValue}" \ + -H "Content-Type: ${contentType}" \ + -H "Authorization: AWS ${s3Key}:${signature}" \ + https://${bucket}.s3.amazonaws.com/${s3_file} +} + +################################### +# Delete s3 object. +# +# Globals: +# None +# Arguments: +# $1 - s3 bucket +# $2 - s3 object key +# $3 - (optional) s3 host suffix +# Returns: +# None +################################### +function s3_delete { + bucket=$1 + s3_file=$2 + resource="/${bucket}/${s3_file}" + contentType="application/octet-stream" + dateValue=`date -R` + stringToSign="DELETE\n\n${contentType}\n${dateValue}\n${resource}" + s3Key=$IT_CASE_S3_ACCESS_KEY + s3Secret=$IT_CASE_S3_SECRET_KEY + signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64` + curl -X DELETE \ + -H "Host: ${bucket}.s3.amazonaws.com" \ + -H "Date: ${dateValue}" \ + -H "Content-Type: ${contentType}" \ + -H "Authorization: AWS ${s3Key}:${signature}" \ + https://${bucket}.s3.amazonaws.com/${s3_file} +} + +################################### +# Delete s3 objects by full path prefix. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - s3 key full path prefix +# Returns: +# None +################################### +function s3_delete_by_full_path_prefix { + AWS_REGION=$AWS_REGION \ + ${s3util} --action deleteByFullPathPrefix --s3prefix "$1" --bucket $IT_CASE_S3_BUCKET +} + +################################### +# Count number of lines in a file of s3 object. +# The lines has to be simple to comply with CSV format +# because SQL is used to query the s3 object. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - s3 file object key +# $2 - s3 bucket +# Returns: +# None +################################### +function s3_get_number_of_lines_in_file { + AWS_REGION=$AWS_REGION \ + ${s3util} --action numberOfLinesInFile --s3file "$1" --bucket $IT_CASE_S3_BUCKET +} + +################################### +# Count number of lines in files of s3 objects filtered by prefix. +# The lines has to be simple to comply with CSV format +# because SQL is used to query the s3 objects. +# +# Globals: +# IT_CASE_S3_BUCKET +# Arguments: +# $1 - s3 key prefix +# $2 - s3 bucket +# $3 - s3 file name prefix w/o directory to filter files by name (optional) +# Returns: +# None +################################### +function s3_get_number_of_lines_by_prefix { + local file_prefix="${3-}" + AWS_REGION=$AWS_REGION \ + ${s3util} --action numberOfLinesInFilesWithFullAndNamePrefix \ + --s3prefix "$1" --s3filePrefix "${file_prefix}" --bucket $IT_CASE_S3_BUCKET +} diff --git a/flink-end-to-end-tests/test-scripts/common_ssl.sh b/flink-end-to-end-tests/test-scripts/common_ssl.sh new file mode 100644 index 0000000000000..a5e19eaf835d0 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/common_ssl.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# NOTE: already sourced in common.sh + +function _set_conf_ssl_helper { + local type=$1 # 'internal' or external 'rest' + local ssl_dir="${TEST_DATA_DIR}/ssl/${type}" + local password="${type}.password" + + if [ "${type}" != "internal" ] && [ "${type}" != "rest" ]; then + echo "Unknown type of ssl connectivity: ${type}. It can be either 'internal' or external 'rest'" + exit 1 + fi + + # clean up the dir that will be used for SSL certificates and trust stores + if [ -e "${ssl_dir}" ]; then + echo "File ${ssl_dir} exists. Deleting it..." + rm -rf "${ssl_dir}" + fi + mkdir -p "${ssl_dir}" + + SANSTRING="dns:${NODENAME}" + for NODEIP in $(get_node_ip) ; do + SANSTRING="${SANSTRING},ip:${NODEIP}" + done + + echo "Using SAN ${SANSTRING}" + + # create certificates + keytool -genkeypair -alias ca -keystore "${ssl_dir}/ca.keystore" -dname "CN=Sample CA" -storepass ${password} -keypass ${password} -keyalg RSA -ext bc=ca:true -storetype PKCS12 + keytool -keystore "${ssl_dir}/ca.keystore" -storepass ${password} -alias ca -exportcert > "${ssl_dir}/ca.cer" + keytool -importcert -keystore "${ssl_dir}/ca.truststore" -alias ca -storepass ${password} -noprompt -file "${ssl_dir}/ca.cer" + + keytool -genkeypair -alias node -keystore "${ssl_dir}/node.keystore" -dname "CN=${NODENAME}" -ext SAN=${SANSTRING} -storepass ${password} -keypass ${password} -keyalg RSA -storetype PKCS12 + keytool -certreq -keystore "${ssl_dir}/node.keystore" -storepass ${password} -alias node -file "${ssl_dir}/node.csr" + keytool -gencert -keystore "${ssl_dir}/ca.keystore" -storepass ${password} -alias ca -ext SAN=${SANSTRING} -infile "${ssl_dir}/node.csr" -outfile "${ssl_dir}/node.cer" + keytool -importcert -keystore "${ssl_dir}/node.keystore" -storepass ${password} -file "${ssl_dir}/ca.cer" -alias ca -noprompt + keytool -importcert -keystore "${ssl_dir}/node.keystore" -storepass ${password} -file "${ssl_dir}/node.cer" -alias node -noprompt + + # keystore is converted into a pem format to use it as node.pem with curl in Flink REST API queries, see also $CURL_SSL_ARGS + openssl pkcs12 -passin pass:${password} -in "${ssl_dir}/node.keystore" -out "${ssl_dir}/node.pem" -nodes + + # adapt config + # (here we rely on security.ssl.enabled enabling SSL for all components and internal as well as + # external communication channels) + set_conf security.ssl.${type}.enabled true + set_conf security.ssl.${type}.keystore ${ssl_dir}/node.keystore + set_conf security.ssl.${type}.keystore-password ${password} + set_conf security.ssl.${type}.key-password ${password} + set_conf security.ssl.${type}.truststore ${ssl_dir}/ca.truststore + set_conf security.ssl.${type}.truststore-password ${password} +} + +function _set_conf_mutual_rest_ssl { + local auth="${1:-server}" # only 'server' or 'mutual' + local mutual="false" + local ssl_dir="${TEST_DATA_DIR}/ssl/rest" + if [ "${auth}" == "mutual" ]; then + CURL_SSL_ARGS="${CURL_SSL_ARGS} --cert ${ssl_dir}/node.pem" + mutual="true"; + fi + echo "Mutual ssl auth: ${mutual}" + set_conf security.ssl.rest.authentication-enabled ${mutual} +} + +function set_conf_rest_ssl { + local auth="${1:-server}" # only 'server' or 'mutual' + local ssl_dir="${TEST_DATA_DIR}/ssl/rest" + _set_conf_ssl_helper "rest" + _set_conf_mutual_rest_ssl ${auth} + REST_PROTOCOL="https" + CURL_SSL_ARGS="${CURL_SSL_ARGS} --cacert ${ssl_dir}/node.pem" +} + +function set_conf_ssl { + local auth="${1:-server}" # only 'server' or 'mutual' + _set_conf_ssl_helper "internal" + set_conf_rest_ssl ${auth} +} diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh index 7b5e50ba4390a..5b98b96e51da1 100755 --- a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh +++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh @@ -124,6 +124,7 @@ elif [ "$1" == "master" ]; then hdfs dfs -chown hadoop-user:hadoop-user /user/hadoop-user kdestroy + echo "Finished master initialization" while true; do sleep 1000; done elif [ "$1" == "worker" ]; then diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml index 9b17acc1656ca..c7736a694899b 100644 --- a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml +++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml @@ -21,6 +21,11 @@ under the License. mapreduce_shuffle + + yarn.nodemanager.vmem-pmem-ratio + 3 + + @@ -33,12 +38,12 @@ under the License. are scheduled on one NM, which wouldn't provoke a previously fixed Kerberos keytab bug. --> yarn.nodemanager.resource.memory-mb - 4100 + 2500 yarn.scheduler.minimum-allocation-mb - 2000 + 1000 diff --git a/flink-end-to-end-tests/test-scripts/elasticsearch-common.sh b/flink-end-to-end-tests/test-scripts/elasticsearch-common.sh index 834e84528b3d3..695905222f553 100644 --- a/flink-end-to-end-tests/test-scripts/elasticsearch-common.sh +++ b/flink-end-to-end-tests/test-scripts/elasticsearch-common.sh @@ -17,8 +17,6 @@ # limitations under the License. ################################################################################ -set -o pipefail - if [[ -z $TEST_DATA_DIR ]]; then echo "Must run common.sh before elasticsearch-common.sh." exit 1 @@ -45,10 +43,10 @@ function wait_elasticsearch_working { echo "Waiting for Elasticsearch node to work..." for ((i=1;i<=60;i++)); do - curl -XGET 'http://localhost:9200' + output=$(curl -XGET 'http://localhost:9200' | grep "cluster_name" || true) # make sure the elasticsearch node is actually working - if [ $? -ne 0 ]; then + if [ "${output}" = "" ]; then sleep 1 else echo "Elasticsearch node is working." @@ -69,7 +67,7 @@ function verify_result_line_number { fi while : ; do - curl "localhost:9200/${index}/_search?q=*&pretty&size=21" > $TEST_DATA_DIR/output + curl "localhost:9200/${index}/_search?q=*&pretty&size=21" > $TEST_DATA_DIR/output || true if [ -n "$(grep "\"total\" : $numRecords" $TEST_DATA_DIR/output)" ]; then echo "Elasticsearch end to end test pass." @@ -85,23 +83,34 @@ function verify_result_hash { local name=$1 local index=$2 local numRecords=$3 - local hash=$4 + local expectedHash=$4 - while : ; do - curl "localhost:9200/${index}/_search?q=*&pretty" > $TEST_DATA_DIR/es_output + for i in {1..30} + do + local error_code=0 - if [ -n "$(grep "\"total\" : $numRecords" $TEST_DATA_DIR/es_output)" ]; then - break - else - echo "Waiting for Elasticsearch records ..." + echo "Result verification attempt $i..." + curl "localhost:9200/${index}/_search?q=*&pretty" > $TEST_DATA_DIR/es_output || true + + # remove meta information + sed '2,9d' $TEST_DATA_DIR/es_output > $TEST_DATA_DIR/es_content + + check_result_hash_no_exit "$name" $TEST_DATA_DIR/es_content "$expectedHash" || error_code=$? + + if [ "$error_code" != "0" ] + then + echo "Result verification attempt $i has failed" sleep 1 + else + break fi done - # remove meta information - sed '2,9d' $TEST_DATA_DIR/es_output > $TEST_DATA_DIR/es_content - - check_result_hash "$name" $TEST_DATA_DIR/es_content "$hash" + if [ "$error_code" != "0" ] + then + echo "All result verification attempts have failed" + exit $error_code + fi } function shutdown_elasticsearch_cluster { diff --git a/flink-end-to-end-tests/test-scripts/kafka-common.sh b/flink-end-to-end-tests/test-scripts/kafka-common.sh index 2dc58f7305a19..7c31534d355f0 100644 --- a/flink-end-to-end-tests/test-scripts/kafka-common.sh +++ b/flink-end-to-end-tests/test-scripts/kafka-common.sh @@ -17,22 +17,24 @@ # limitations under the License. ################################################################################ -set -o pipefail - if [[ -z $TEST_DATA_DIR ]]; then echo "Must run common.sh before kafka-common.sh." exit 1 fi -KAFKA_DIR=$TEST_DATA_DIR/kafka_2.11-0.10.2.0 -CONFLUENT_DIR=$TEST_DATA_DIR/confluent-3.2.0 +KAFKA_VERSION="$1" +CONFLUENT_VERSION="$2" +CONFLUENT_MAJOR_VERSION="$3" + +KAFKA_DIR=$TEST_DATA_DIR/kafka_2.11-$KAFKA_VERSION +CONFLUENT_DIR=$TEST_DATA_DIR/confluent-$CONFLUENT_VERSION SCHEMA_REGISTRY_PORT=8082 SCHEMA_REGISTRY_URL=http://localhost:${SCHEMA_REGISTRY_PORT} function setup_kafka_dist { # download Kafka mkdir -p $TEST_DATA_DIR - KAFKA_URL="https://archive.apache.org/dist/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz" + KAFKA_URL="https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz" echo "Downloading Kafka from $KAFKA_URL" curl "$KAFKA_URL" > $TEST_DATA_DIR/kafka.tgz @@ -46,7 +48,7 @@ function setup_kafka_dist { function setup_confluent_dist { # download confluent mkdir -p $TEST_DATA_DIR - CONFLUENT_URL="http://packages.confluent.io/archive/3.2/confluent-oss-3.2.0-2.11.tar.gz" + CONFLUENT_URL="http://packages.confluent.io/archive/$CONFLUENT_MAJOR_VERSION/confluent-oss-$CONFLUENT_VERSION-2.11.tar.gz" echo "Downloading confluent from $CONFLUENT_URL" curl "$CONFLUENT_URL" > $TEST_DATA_DIR/confluent.tgz @@ -76,16 +78,18 @@ function stop_kafka_cluster { $KAFKA_DIR/bin/kafka-server-stop.sh $KAFKA_DIR/bin/zookeeper-server-stop.sh - PIDS=$(jps -vl | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}') + # Terminate Kafka process if it still exists + PIDS=$(jps -vl | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}'|| echo "") if [ ! -z "$PIDS" ]; then - kill -s TERM $PIDS + kill -s TERM $PIDS || true fi - PIDS=$(jps -vl | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}') + # Terminate QuorumPeerMain process if it still exists + PIDS=$(jps -vl | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}'|| echo "") if [ ! -z "$PIDS" ]; then - kill -s TERM $PIDS + kill -s TERM $PIDS || true fi } @@ -120,14 +124,7 @@ function get_partition_end_offset { local topic=$1 local partition=$2 - # first, use the console consumer to produce a dummy consumer group - read_messages_from_kafka 0 $topic dummy-consumer - - # then use the consumer offset utility to get the LOG_END_OFFSET value for the specified partition - $KAFKA_DIR/bin/kafka-consumer-groups.sh --describe --group dummy-consumer --bootstrap-server localhost:9092 2> /dev/null \ - | grep "$topic \+$partition" \ - | tr -s " " \ - | cut -d " " -f 4 + $KAFKA_DIR/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic $topic --partitions $partition --time -1 | cut -d ":" -f 3 } function start_confluent_schema_registry { diff --git a/flink-end-to-end-tests/test-scripts/kafka_sql_common.sh b/flink-end-to-end-tests/test-scripts/kafka_sql_common.sh new file mode 100644 index 0000000000000..468f0587db757 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/kafka_sql_common.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +KAFKA_CONNECTOR_VERSION="$1" +KAFKA_VERSION="$2" +CONFLUENT_VERSION="$3" +CONFLUENT_MAJOR_VERSION="$4" +KAFKA_SQL_VERSION="$5" + +source "$(dirname "$0")"/kafka-common.sh $2 $3 $4 + +function create_kafka_json_source { + topicName="$1" + create_kafka_topic 1 1 $topicName + + # put JSON data into Kafka + echo "Sending messages to Kafka..." + + send_messages_to_kafka '{"timestamp": "2018-03-12 08:00:00", "user": "Alice", "event": { "type": "WARNING", "message": "This is a warning."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 08:10:00", "user": "Alice", "event": { "type": "WARNING", "message": "This is a warning."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 09:00:00", "user": "Bob", "event": { "type": "WARNING", "message": "This is another warning."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 09:10:00", "user": "Alice", "event": { "type": "INFO", "message": "This is a info."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 09:20:00", "user": "Steve", "event": { "type": "INFO", "message": "This is another info."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 09:30:00", "user": "Steve", "event": { "type": "INFO", "message": "This is another info."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 09:30:00", "user": null, "event": { "type": "WARNING", "message": "This is a bad message because the user is missing."}}' $topicName + send_messages_to_kafka '{"timestamp": "2018-03-12 10:40:00", "user": "Bob", "event": { "type": "ERROR", "message": "This is an error."}}' $topicName +} + +function get_kafka_json_source_schema { + topicName="$1" + tableName="$2" + cat << EOF + - name: $tableName + type: source-table + update-mode: append + schema: + - name: rowtime + type: TIMESTAMP + rowtime: + timestamps: + type: from-field + from: timestamp + watermarks: + type: periodic-bounded + delay: 2000 + - name: user + type: VARCHAR + - name: event + type: ROW + connector: + type: kafka + version: "$KAFKA_SQL_VERSION" + topic: $topicName + startup-mode: earliest-offset + properties: + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + format: + type: json + json-schema: > + { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "user": { + "type": ["string", "null"] + }, + "event": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + } + } +EOF +} diff --git a/flink-end-to-end-tests/test-scripts/test-runner-common.sh b/flink-end-to-end-tests/test-scripts/test-runner-common.sh index 0e56d2ac250d4..1eef75c5acd85 100644 --- a/flink-end-to-end-tests/test-scripts/test-runner-common.sh +++ b/flink-end-to-end-tests/test-scripts/test-runner-common.sh @@ -29,7 +29,7 @@ source "${END_TO_END_DIR}"/test-scripts/common.sh function run_test { local description="$1" local command="$2" - local skip_check_exceptions="$3" + local skip_check_exceptions=${3:-} printf "\n==============================================================================\n" printf "Running '${description}'\n" @@ -41,14 +41,32 @@ function run_test { backup_config start_timer + + function test_error() { + echo "[FAIL] Test script contains errors." + post_test_validation 1 "$description" "$skip_check_exceptions" + } + trap 'test_error' ERR + ${command} exit_code="$?" - time_elapsed=$(end_timer) + post_test_validation ${exit_code} "$description" "$skip_check_exceptions" +} + +# Validates the test result and exit code after its execution. +function post_test_validation { + local exit_code="$1" + local description="$2" + local skip_check_exceptions="$3" + + local time_elapsed=$(end_timer) if [[ "${skip_check_exceptions}" != "skip_check_exceptions" ]]; then check_logs_for_errors check_logs_for_exceptions check_logs_for_non_empty_out_files + else + echo "Checking of logs skipped." fi # Investigate exit_code for failures of test executable as well as EXIT_CODE for failures of the test. @@ -83,7 +101,8 @@ function cleanup_proc { # Cleans up all temporary folders and files function cleanup_tmp_files { - clean_log_files + rm ${FLINK_DIR}/log/* + echo "Deleted all files under ${FLINK_DIR}/log/" rm -rf ${TEST_DATA_DIR} 2> /dev/null echo "Deleted ${TEST_DATA_DIR}" diff --git a/flink-end-to-end-tests/test-scripts/test_batch_allround.sh b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh index f8f591f7dc0be..502c12f7dbeaa 100755 --- a/flink-end-to-end-tests/test-scripts/test_batch_allround.sh +++ b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh @@ -17,6 +17,8 @@ # limitations under the License. ################################################################################ +set -Eeuo pipefail + source "$(dirname "$0")"/common.sh TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-dataset-allround-test/target/DataSetAllroundTestProgram.jar @@ -27,7 +29,7 @@ echo "Run DataSet-Allround-Test Program" echo "taskmanager.network.memory.min: 10485760" >> $FLINK_DIR/conf/flink-conf.yaml echo "taskmanager.network.memory.max: 10485760" >> $FLINK_DIR/conf/flink-conf.yaml -set_conf_ssl +set_conf_ssl "server" start_cluster start_taskmanagers 3 diff --git a/flink-end-to-end-tests/test-scripts/test_cli.sh b/flink-end-to-end-tests/test-scripts/test_cli.sh new file mode 100755 index 0000000000000..b9d285b635204 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_cli.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -Eeuo pipefail + +source "$(dirname "$0")"/common.sh + +TEST_PROGRAM_JAR=$END_TO_END_DIR/flink-cli-test/target/PeriodicStreamingJob.jar + +start_cluster +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start + +# CLI regular expressions +JOB_ID_REGEX_EXTRACTOR=".*JobID ([0-9,a-f]*)" +JOB_INFO_PACT_DATA_SOURCE_REGEX_EXTRACTOR="\"pact\": \"(Data Source)\"" +JOB_INFO_PACT_DATA_SINK_REGEX_EXTRACTOR="\"pact\": \"(Data Sink)\"" +JOB_LIST_REGEX_EXTRACTOR_BY_STATUS="([0-9,a-f]*) :" + +EXIT_CODE=0 + +function extract_job_id_from_job_submission_return() { + if [[ $1 =~ $JOB_ID_REGEX_EXTRACTOR ]]; + then + JOB_ID="${BASH_REMATCH[1]}"; + else + JOB_ID="" + fi + echo "$JOB_ID" +} + +function extract_valid_pact_from_job_info_return() { + PACT_MATCH=0 + if [[ $1 =~ $JOB_INFO_PACT_DATA_SOURCE_REGEX_EXTRACTOR ]]; + then + PACT_MATCH=$PACT_MATCH + else + PACT_MATCH=-1 + fi + if [[ $1 =~ $JOB_INFO_PACT_DATA_SINK_REGEX_EXTRACTOR ]]; + then + PACT_MATCH=$PACT_MATCH + else + PACT_MATCH=-1 + fi + echo ${PACT_MATCH} +} + +function extract_valid_job_list_by_type_from_job_list_return() { + JOB_LIST_MATCH=0 + JOB_LIST_REGEX_EXTRACTOR="$JOB_LIST_REGEX_EXTRACTOR_BY_STATUS $2 $3" + if [[ $1 =~ $JOB_LIST_REGEX_EXTRACTOR ]]; + then + JOB_LIST_MATCH=$JOB_LIST_MATCH + else + JOB_LIST_MATCH=-1 + fi + echo ${JOB_LIST_MATCH} +} + +function extract_task_manager_slot_request_count() { + COUNT=`grep "Receive slot request" $FLINK_DIR/log/*taskexecutor*.log | wc -l` + echo $COUNT +} + +printf "\n==============================================================================\n" +printf "Test default job launch with non-detach mode\n" +printf "==============================================================================\n" +RESULT=`$FLINK_DIR/bin/flink run $FLINK_DIR/examples/batch/WordCount.jar` +EXIT_CODE=$? +echo "$RESULT" + +if [[ $RESULT != *"(java.util.ArrayList) [170 elements]"* ]];then + echo "[FAIL] Invalid accumulator result." + EXIT_CODE=1 +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Test job launch with complex parameter set\n" + printf "==============================================================================\n" + eval "$FLINK_DIR/bin/flink run -m localhost:8081 -p 4 -q \ + -c org.apache.flink.examples.java.wordcount.WordCount \ + $FLINK_DIR/examples/batch/WordCount.jar \ + --input file:///$FLINK_DIR/README.txt \ + --output file:///${TEST_DATA_DIR}/result1" + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + ROW_COUNT=`cat ${TEST_DATA_DIR}/result1/* | wc -l` + if [ $((ROW_COUNT)) -ne 111 ]; then + echo "[FAIL] Unexpected number of rows in output." + echo "Found: $ROW_COUNT" + EXIT_CODE=1 + fi +fi + +if [ $EXIT_CODE == 0 ]; then + RECEIVED_TASKMGR_REQUEST=`extract_task_manager_slot_request_count` + # expected 1 from default launch and 4 from complex parameter set. + if [[ $RECEIVED_TASKMGR_REQUEST != 5 ]]; then + echo "[FAIL] Unexpected task manager slot count." + echo "Received slots: $RECEIVED_TASKMGR_REQUEST" + EXIT_CODE=1 + fi +fi + +printf "\n==============================================================================\n" +printf "Test CLI information\n" +printf "==============================================================================\n" +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink info $FLINK_DIR/examples/batch/WordCount.jar` + echo "$RETURN" + PACT_MATCH=`extract_valid_pact_from_job_info_return "$RETURN"` + if [[ $PACT_MATCH == -1 ]]; then # expect at least a Data Source and a Data Sink pact match + echo "[FAIL] Data source and/or sink are missing." + EXIT_CODE=1 + fi +fi + +printf "\n==============================================================================\n" +printf "Test operation on running streaming jobs\n" +printf "==============================================================================\n" +JOB_ID="" +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink run -d $TEST_PROGRAM_JAR --outputPath file:///${TEST_DATA_DIR}/result2` + echo "$RETURN" + JOB_ID=`extract_job_id_from_job_submission_return "$RETURN"` + EXIT_CODE=$? # expect matching job id extraction +fi + +printf "\n==============================================================================\n" +printf "Test list API on a streaming job \n" +printf "==============================================================================\n" +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink list -a` + echo "$RETURN" + JOB_LIST_MATCH=`extract_valid_job_list_by_type_from_job_list_return "$RETURN" "Flink Streaming Job" ""` + if [[ $JOB_LIST_MATCH == -1 ]]; then # expect match for all job + echo "[FAIL] Unexpected 'Flink Streaming Job' list." + EXIT_CODE=1 + fi +fi + +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink list -r` + echo "$RETURN" + JOB_LIST_MATCH=`extract_valid_job_list_by_type_from_job_list_return "$RETURN" "Flink Streaming Job" "\(RUNNING\)"` + if [[ $JOB_LIST_MATCH == -1 ]]; then # expect match for running job + echo "[FAIL] Unexpected 'Flink Streaming Job' 'RUNNING' list." + EXIT_CODE=1 + fi +fi + +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink list -s` + echo "$RETURN" + JOB_LIST_MATCH=`extract_valid_job_list_by_type_from_job_list_return "$RETURN" "Flink Streaming Job" "\(CREATED\)"` + if [[ $JOB_LIST_MATCH != -1 ]]; then # expect no match for scheduled job + echo "[FAIL] Unexpected 'Flink Streaming Job' 'CREATED' list." + EXIT_CODE=1 + fi +fi + +printf "\n==============================================================================\n" +printf "Test canceling a running streaming jobs\n" +printf "==============================================================================\n" +if [ $EXIT_CODE == 0 ]; then + eval "$FLINK_DIR/bin/flink cancel ${JOB_ID}" + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + RETURN=`$FLINK_DIR/bin/flink list -a` + echo "$RETURN" + JOB_LIST_MATCH=`extract_valid_job_list_by_type_from_job_list_return "$RETURN" "Flink Streaming Job" "\(CANCELED\)"` + if [[ $JOB_LIST_MATCH == -1 ]]; then # expect match for canceled job + echo "[FAIL] Unexpected 'Flink Streaming Job' 'CANCELED' list." + EXIT_CODE=1 + fi +fi + +exit $EXIT_CODE diff --git a/flink-end-to-end-tests/test-scripts/test_confluent_schema_registry.sh b/flink-end-to-end-tests/test-scripts/test_confluent_schema_registry.sh index 323234fa683d0..b1500a0a96a62 100755 --- a/flink-end-to-end-tests/test-scripts/test_confluent_schema_registry.sh +++ b/flink-end-to-end-tests/test-scripts/test_confluent_schema_registry.sh @@ -17,8 +17,10 @@ # limitations under the License. ################################################################################ +set -Eeuo pipefail + source "$(dirname "$0")"/common.sh -source "$(dirname "$0")"/kafka-common.sh +source "$(dirname "$0")"/kafka-common.sh 0.10.2.0 3.2.0 3.2 function verify_output { local expected=$(printf $1) diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh index 370ef052a4d4e..2d8aa4f47d415 100755 --- a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh +++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh @@ -21,6 +21,8 @@ source "$(dirname "$0")"/common.sh DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts +DOCKER_IMAGE_BUILD_RETRIES=3 +BUILD_BACKOFF_TIME=5 export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job @@ -30,12 +32,19 @@ export INPUT_PATH=/data/test/input export OUTPUT_PATH=/data/test/output export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out" -# user inside the container must be able to createto workaround in-container permissions +build_image() { + ./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME} +} + +# user inside the container must be able to create files, this is a workaround in-container permissions mkdir -p $OUTPUT_VOLUME chmod 777 $OUTPUT_VOLUME cd "$DOCKER_MODULE_DIR" -./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME} +if ! retry_times $DOCKER_IMAGE_BUILD_RETRIES ${BUILD_BACKOFF_TIME} build_image; then + echo "Failed to build docker image. Aborting..." + exit 1 +fi cd "$END_TO_END_DIR" docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null diff --git a/flink-end-to-end-tests/test-scripts/test_heavy_deployment.sh b/flink-end-to-end-tests/test-scripts/test_heavy_deployment.sh new file mode 100755 index 0000000000000..cc1a64d9623ef --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_heavy_deployment.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +CHECKPOINT_DIR="file://$TEST_DATA_DIR/savepoint-e2e-test-chckpt-dir" + +TEST=flink-heavy-deployment-stress-test +TEST_PROGRAM_NAME=HeavyDeploymentStressTestProgram +TEST_PROGRAM_JAR=${END_TO_END_DIR}/$TEST/target/$TEST_PROGRAM_NAME.jar + +set_conf "taskmanager.heap.mb" "512" # 512Mb x 10TMs = 5Gb total heap + +set_conf "taskmanager.memory.size" "8" # 8Mb +set_conf "taskmanager.network.memory.min" "8mb" +set_conf "taskmanager.network.memory.max" "8mb" +set_conf "taskmanager.network.request-backoff.max" "60000" +set_conf "taskmanager.memory.segment-size" "8kb" + +set_conf "taskmanager.numberOfTaskSlots" "10" # 10 slots per TM + +start_cluster # this also starts 1TM +start_taskmanagers 9 # 1TM + 9TM = 10TM a 10 slots = 100 slots + +# This call will result in a deployment with state meta data of 100 x 100 x 50 union states x each 50 entries. +# We can scale up the numbers to make the test even heavier. +$FLINK_DIR/bin/flink run ${TEST_PROGRAM_JAR} \ +--environment.max_parallelism 1024 --environment.parallelism 100 \ +--environment.restart_strategy fixed_delay --environment.restart_strategy.fixed_delay.attempts 3 \ +--state_backend.checkpoint_directory ${CHECKPOINT_DIR} \ +--heavy_deployment_test.num_list_states_per_op 50 --heavy_deployment_test.num_partitions_per_list_state 50 diff --git a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh index 582b5a0af0d5f..93668d87a2f79 100755 --- a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh +++ b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh @@ -34,6 +34,7 @@ set_conf "taskmanager.memory.segment-size" "8kb" set_conf "taskmanager.network.netty.server.numThreads" "1" set_conf "taskmanager.network.netty.client.numThreads" "1" +set_conf "taskmanager.network.request-backoff.max" "60000" set_conf "taskmanager.numberOfTaskSlots" "1" diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh index 4529fd904a687..96320fd53c50c 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh @@ -22,6 +22,8 @@ source "$(dirname "$0")"/common.sh DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker KUBERNETES_MODULE_DIR=${END_TO_END_DIR}/../flink-container/kubernetes CONTAINER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts +MINIKUBE_START_RETRIES=3 +MINIKUBE_START_BACKOFF=5 export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount export FLINK_IMAGE_NAME=test_kubernetes_embedded_job @@ -37,10 +39,29 @@ function cleanup { rm -rf ${OUTPUT_VOLUME} } +function check_kubernetes_status { + local status=`minikube status` + echo ${status} | grep -q "minikube: Running cluster: Running kubectl: Correctly Configured" + return $? +} + +function start_kubernetes_if_not_running { + if ! check_kubernetes_status; then + minikube start + fi + + return $(check_kubernetes_status) +} + trap cleanup EXIT mkdir -p $OUTPUT_VOLUME +if ! retry_times ${MINIKUBE_START_RETRIES} ${MINIKUBE_START_BACKOFF} start_kubernetes_if_not_running; then + echo "Minikube not running. Could not start minikube. Aborting..." + exit 1 +fi + eval $(minikube docker-env) cd "$DOCKER_MODULE_DIR" ./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_IMAGE_NAME} diff --git a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh index 56d811e14a1a9..db0174a2160cf 100755 --- a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh +++ b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh @@ -86,6 +86,7 @@ function run_test() { fi kill_random_taskmanager + wait_for_number_of_running_tms 0 latest_snapshot_count=$(cat $FLINK_DIR/log/*out* | grep "on snapshot" | tail -n 1 | awk '{print $4}') echo "Latest snapshot count was ${latest_snapshot_count}" diff --git a/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh b/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh index e1ba65d70c3b1..48d68c98e9d68 100755 --- a/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh +++ b/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh @@ -66,8 +66,9 @@ TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-datastream-allround-test/target/DataStr function buildBaseJobCmd { local dop=$1 + local flink_args="$2" - echo "$FLINK_DIR/bin/flink run -d -p $dop $TEST_PROGRAM_JAR \ + echo "$FLINK_DIR/bin/flink run -d ${flink_args} -p $dop $TEST_PROGRAM_JAR \ --test.semantics exactly-once \ --environment.parallelism $dop \ --environment.externalize_checkpoint true \ @@ -99,35 +100,45 @@ fi DATASTREAM_JOB=$($JOB_CMD | grep "Job has been submitted with JobID" | sed 's/.* //g') -wait_job_running $DATASTREAM_JOB - if [[ $SIMULATE_FAILURE == "true" ]]; then wait_job_terminal_state $DATASTREAM_JOB FAILED else + wait_job_running $DATASTREAM_JOB wait_num_checkpoints $DATASTREAM_JOB 1 wait_oper_metric_num_in_records SemanticsCheckMapper.0 200 cancel_job $DATASTREAM_JOB fi -CHECKPOINT_PATH=$(ls -d $CHECKPOINT_DIR/$DATASTREAM_JOB/chk-[1-9]*) +# take the latest checkpoint +CHECKPOINT_PATH=$(find_latest_completed_checkpoint ${CHECKPOINT_DIR}/${DATASTREAM_JOB}) if [ -z $CHECKPOINT_PATH ]; then echo "Expected an externalized checkpoint to be present, but none exists." exit 1 fi -NUM_CHECKPOINTS=$(echo $CHECKPOINT_PATH | wc -l | tr -d ' ') -if (( $NUM_CHECKPOINTS > 1 )); then - echo "Expected only exactly 1 externalized checkpoint to be present, but $NUM_CHECKPOINTS exists." - exit 1 -fi - echo "Restoring job with externalized checkpoint at $CHECKPOINT_PATH ..." -BASE_JOB_CMD=`buildBaseJobCmd $NEW_DOP` +BASE_JOB_CMD=`buildBaseJobCmd $NEW_DOP "-s file://${CHECKPOINT_PATH}"` +JOB_CMD="" +if [[ $SIMULATE_FAILURE == "true" ]]; then + JOB_CMD="$BASE_JOB_CMD \ + --test.simulate_failure true \ + --test.simulate_failure.num_records 0 \ + --test.simulate_failure.num_checkpoints 0 \ + --test.simulate_failure.max_failures 0 \ + --environment.restart_strategy no_restart" +else + JOB_CMD=$BASE_JOB_CMD +fi + +DATASTREAM_JOB=$($JOB_CMD | grep "Job has been submitted with JobID" | sed 's/.* //g') -DATASTREAM_JOB=$($BASE_JOB_CMD | grep "Job has been submitted with JobID" | sed 's/.* //g') +if [ -z $DATASTREAM_JOB ]; then + echo "Resuming from externalized checkpoint job could not be started." + exit 1 +fi wait_job_running $DATASTREAM_JOB wait_oper_metric_num_in_records SemanticsCheckMapper.0 200 diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh index e5ac5bcecdce9..ddbb6868424dd 100755 --- a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh +++ b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh @@ -19,33 +19,12 @@ # Tests for our shaded/bundled Hadoop S3A file system. -if [[ -z "$ARTIFACTS_AWS_BUCKET" ]]; then - echo "Did not find AWS environment variables, NOT running Shaded Hadoop S3A e2e tests." - exit 0 -else - echo "Found AWS bucket $ARTIFACTS_AWS_BUCKET, running Shaded Hadoop S3A e2e tests." -fi - source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/common_s3.sh -s3_put $TEST_INFRA_DIR/test-data/words $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-s3a -# make sure we delete the file at the end -function s3_cleanup { - s3_delete $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-s3a - rm $FLINK_DIR/lib/flink-s3-fs*.jar - - # remove any leftover settings - sed -i -e 's/s3.access-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" - sed -i -e 's/s3.secret-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" -} -trap s3_cleanup EXIT - -cp $FLINK_DIR/opt/flink-s3-fs-hadoop-*.jar $FLINK_DIR/lib/ -echo "s3.access-key: $ARTIFACTS_AWS_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" -echo "s3.secret-key: $ARTIFACTS_AWS_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" - +s3_setup hadoop start_cluster -$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input s3:/$resource --output $TEST_DATA_DIR/out/wc_out +$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out -check_result_hash "WordCountWithShadedS3A" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5" \ No newline at end of file +check_result_hash "WordCountWithShadedS3A" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5" diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh index 4092805dc278f..9ebbb0d8c08fe 100755 --- a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh +++ b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh @@ -19,33 +19,12 @@ # Tests for our shaded/bundled Hadoop S3A file system. -if [[ -z "$ARTIFACTS_AWS_BUCKET" ]]; then - echo "Did not find AWS environment variables, NOT running Shaded Presto S3 e2e tests." - exit 0 -else - echo "Found AWS bucket $ARTIFACTS_AWS_BUCKET, running Shaded Presto S3 e2e tests." -fi - source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/common_s3.sh -s3_put $TEST_INFRA_DIR/test-data/words $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-presto-s3 -# make sure we delete the file at the end -function s3_cleanup { - s3_delete $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-presto-s3 - rm $FLINK_DIR/lib/flink-s3-fs*.jar -} -trap s3_cleanup EXIT - -cp $FLINK_DIR/opt/flink-s3-fs-presto-*.jar $FLINK_DIR/lib/ -echo "s3.access-key: $ARTIFACTS_AWS_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" -echo "s3.secret-key: $ARTIFACTS_AWS_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml" - +s3_setup presto start_cluster -$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input s3:/$resource --output $TEST_DATA_DIR/out/wc_out +$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out check_result_hash "WordCountWithShadedPrestoS3" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5" - -# remove any leftover settings -sed -i -e 's/s3.access-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" -sed -i -e 's/s3.secret-key: .*//' "$FLINK_DIR/conf/flink-conf.yaml" diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client.sh b/flink-end-to-end-tests/test-scripts/test_sql_client.sh index ce3d194487766..8ace250a31900 100755 --- a/flink-end-to-end-tests/test-scripts/test_sql_client.sh +++ b/flink-end-to-end-tests/test-scripts/test_sql_client.sh @@ -17,8 +17,21 @@ # limitations under the License. ################################################################################ +set -Eeuo pipefail + +KAFKA_CONNECTOR_VERSION="2.0" +KAFKA_VERSION="2.0.1" +CONFLUENT_VERSION="5.0.0" +CONFLUENT_MAJOR_VERSION="5.0" +KAFKA_SQL_VERSION="universal" + source "$(dirname "$0")"/common.sh -source "$(dirname "$0")"/kafka-common.sh +source "$(dirname "$0")"/kafka_sql_common.sh \ + $KAFKA_CONNECTOR_VERSION \ + $KAFKA_VERSION \ + $CONFLUENT_VERSION \ + $CONFLUENT_MAJOR_VERSION \ + $KAFKA_SQL_VERSION source "$(dirname "$0")"/elasticsearch-common.sh SQL_TOOLBOX_JAR=$END_TO_END_DIR/flink-sql-client-test/target/SqlToolbox.jar @@ -34,7 +47,7 @@ mkdir -p $EXTRACTED_JAR for SQL_JAR in $SQL_JARS_DIR/*.jar; do echo "Checking SQL JAR: $SQL_JAR" - unzip $SQL_JAR -d $EXTRACTED_JAR > /dev/null + (cd $EXTRACTED_JAR && jar xf $SQL_JAR) # check for proper shading for EXTRACTED_FILE in $(find $EXTRACTED_JAR -type f); do @@ -58,6 +71,8 @@ for SQL_JAR in $SQL_JARS_DIR/*.jar; do rm -r $EXTRACTED_JAR/* done +rm -r $EXTRACTED_JAR + ################################################################################ # Prepare connectors ################################################################################ @@ -83,25 +98,9 @@ setup_kafka_dist start_kafka_cluster -create_kafka_topic 1 1 test-json +create_kafka_json_source test-json create_kafka_topic 1 1 test-avro -# put JSON data into Kafka -echo "Sending messages to Kafka..." - -send_messages_to_kafka '{"timestamp": "2018-03-12 08:00:00", "user": "Alice", "event": { "type": "WARNING", "message": "This is a warning."}}' test-json -# duplicate -send_messages_to_kafka '{"timestamp": "2018-03-12 08:10:00", "user": "Alice", "event": { "type": "WARNING", "message": "This is a warning."}}' test-json -send_messages_to_kafka '{"timestamp": "2018-03-12 09:00:00", "user": "Bob", "event": { "type": "WARNING", "message": "This is another warning."}}' test-json -send_messages_to_kafka '{"timestamp": "2018-03-12 09:10:00", "user": "Alice", "event": { "type": "INFO", "message": "This is a info."}}' test-json -send_messages_to_kafka '{"timestamp": "2018-03-12 09:20:00", "user": "Steve", "event": { "type": "INFO", "message": "This is another info."}}' test-json -# duplicate -send_messages_to_kafka '{"timestamp": "2018-03-12 09:30:00", "user": "Steve", "event": { "type": "INFO", "message": "This is another info."}}' test-json -# filtered in results -send_messages_to_kafka '{"timestamp": "2018-03-12 09:30:00", "user": null, "event": { "type": "WARNING", "message": "This is a bad message because the user is missing."}}' test-json -# pending in results -send_messages_to_kafka '{"timestamp": "2018-03-12 10:40:00", "user": "Bob", "event": { "type": "ERROR", "message": "This is an error."}}' test-json - # prepare Elasticsearch echo "Preparing Elasticsearch..." @@ -112,141 +111,35 @@ setup_elasticsearch $DOWNLOAD_URL wait_elasticsearch_working ################################################################################ -# Run a SQL statement +# Prepare Flink ################################################################################ -echo "Testing SQL statement..." - -# prepare Flink echo "Preparing Flink..." start_cluster start_taskmanagers 2 +################################################################################ +# Run SQL statements +################################################################################ + +echo "Testing SQL statements..." + +JSON_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "json" ) +KAFKA_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "kafka_" ) +ELASTICSEARCH_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "elasticsearch6" ) + # create session environment file RESULT=$TEST_DATA_DIR/result SQL_CONF=$TEST_DATA_DIR/sql-client-session.conf -cat > $SQL_CONF << EOF +cat >> $SQL_CONF << EOF tables: - - name: JsonSourceTable - type: source-table - update-mode: append - schema: - - name: rowtime - type: TIMESTAMP - rowtime: - timestamps: - type: from-field - from: timestamp - watermarks: - type: periodic-bounded - delay: 2000 - - name: user - type: VARCHAR - - name: event - type: ROW - connector: - type: kafka - version: "0.10" - topic: test-json - startup-mode: earliest-offset - properties: - - key: zookeeper.connect - value: localhost:2181 - - key: bootstrap.servers - value: localhost:9092 - format: - type: json - json-schema: > - { - "type": "object", - "properties": { - "timestamp": { - "type": "string" - }, - "user": { - "type": ["string", "null"] - }, - "event": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "message": { - "type": "string" - } - } - } - } - } - - name: AvroBothTable - type: source-sink-table - update-mode: append - schema: - - name: event_timestamp - type: VARCHAR - - name: user - type: VARCHAR - - name: message - type: VARCHAR - - name: duplicate_count - type: BIGINT - connector: - type: kafka - version: "0.10" - topic: test-avro - startup-mode: earliest-offset - properties: - - key: zookeeper.connect - value: localhost:2181 - - key: bootstrap.servers - value: localhost:9092 - format: - type: avro - avro-schema: > - { - "namespace": "org.apache.flink.table.tests", - "type": "record", - "name": "NormalizedEvent", - "fields": [ - {"name": "event_timestamp", "type": "string"}, - {"name": "user", "type": ["string", "null"]}, - {"name": "message", "type": "string"}, - {"name": "duplicate_count", "type": "long"} - ] - } - - name: CsvSinkTable - type: sink-table - update-mode: append - schema: - - name: event_timestamp - type: VARCHAR - - name: user - type: VARCHAR - - name: message - type: VARCHAR - - name: duplicate_count - type: BIGINT - - name: constant - type: VARCHAR - connector: - type: filesystem - path: $RESULT - format: - type: csv - fields: - - name: event_timestamp - type: VARCHAR - - name: user - type: VARCHAR - - name: message - type: VARCHAR - - name: duplicate_count - type: BIGINT - - name: constant - type: VARCHAR +EOF + +get_kafka_json_source_schema test-json JsonSourceTable >> $SQL_CONF + +cat >> $SQL_CONF << EOF - name: ElasticsearchUpsertSinkTable type: sink-table update-mode: upsert @@ -304,69 +197,16 @@ EOF # submit SQL statements -echo "Executing SQL: Kafka JSON -> Kafka Avro" - -read -r -d '' SQL_STATEMENT_1 << EOF -INSERT INTO AvroBothTable - SELECT - CAST(TUMBLE_START(rowtime, INTERVAL '1' HOUR) AS VARCHAR) AS event_timestamp, - user, - RegReplace(event.message, ' is ', ' was ') AS message, - COUNT(*) AS duplicate_count - FROM JsonSourceTable - WHERE user IS NOT NULL - GROUP BY - user, - event.message, - TUMBLE(rowtime, INTERVAL '1' HOUR) -EOF - -echo "$SQL_STATEMENT_1" - -$FLINK_DIR/bin/sql-client.sh embedded \ - --library $SQL_JARS_DIR \ - --jar $SQL_TOOLBOX_JAR \ - --environment $SQL_CONF \ - --update "$SQL_STATEMENT_1" - -echo "Executing SQL: Kafka Avro -> Filesystem CSV" - -read -r -d '' SQL_STATEMENT_2 << EOF -INSERT INTO CsvSinkTable - SELECT AvroBothTable.*, RegReplace('Test constant folding.', 'Test', 'Success') AS constant - FROM AvroBothTable -EOF - -echo "$SQL_STATEMENT_2" - -$FLINK_DIR/bin/sql-client.sh embedded \ - --library $SQL_JARS_DIR \ - --jar $SQL_TOOLBOX_JAR \ - --environment $SQL_CONF \ - --update "$SQL_STATEMENT_2" - -echo "Waiting for CSV results..." -for i in {1..10}; do - if [ -e $RESULT ]; then - CSV_LINE_COUNT=`cat $RESULT | wc -l` - if [ $((CSV_LINE_COUNT)) -eq 4 ]; then - break - fi - fi - sleep 5 -done - -check_result_hash "SQL Client Kafka" $RESULT "0a1bf8bf716069b7269f575f87a802c0" - echo "Executing SQL: Values -> Elasticsearch (upsert)" -read -r -d '' SQL_STATEMENT_3 << EOF +SQL_STATEMENT_3=$(cat << EOF INSERT INTO ElasticsearchUpsertSinkTable SELECT user_id, user_name, COUNT(*) AS user_count FROM (VALUES (1, 'Bob'), (22, 'Alice'), (42, 'Greg'), (42, 'Greg'), (42, 'Greg'), (1, 'Bob')) AS UserCountTable(user_id, user_name) GROUP BY user_id, user_name EOF +) JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \ --library $SQL_JARS_DIR \ @@ -380,7 +220,7 @@ verify_result_hash "SQL Client Elasticsearch Upsert" "$ELASTICSEARCH_INDEX" 3 "9 echo "Executing SQL: Values -> Elasticsearch (append, no key)" -read -r -d '' SQL_STATEMENT_4 << EOF +SQL_STATEMENT_4=$(cat << EOF INSERT INTO ElasticsearchAppendSinkTable SELECT * FROM ( @@ -393,9 +233,12 @@ INSERT INTO ElasticsearchAppendSinkTable (1, 'Bob', CAST(0 AS BIGINT))) AS UserCountTable(user_id, user_name, user_count) EOF +) JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \ - --library $SQL_JARS_DIR \ + --jar $KAFKA_SQL_JAR \ + --jar $JSON_SQL_JAR \ + --jar $ELASTICSEARCH_SQL_JAR \ --jar $SQL_TOOLBOX_JAR \ --environment $SQL_CONF \ --update "$SQL_STATEMENT_4" | grep "Job ID:" | sed 's/.* //g') @@ -407,7 +250,7 @@ verify_result_line_number 9 "$ELASTICSEARCH_INDEX" echo "Executing SQL: Match recognize -> Elasticsearch" -read -r -d '' SQL_STATEMENT_5 << EOF +SQL_STATEMENT_5=$(cat << EOF INSERT INTO ElasticsearchAppendSinkTable SELECT 1 as user_id, T.userName as user_name, cast(1 as BIGINT) as user_count FROM ( @@ -423,9 +266,12 @@ INSERT INTO ElasticsearchAppendSinkTable A as user = 'Alice' ) T EOF +) JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \ - --library $SQL_JARS_DIR \ + --jar $KAFKA_SQL_JAR \ + --jar $JSON_SQL_JAR \ + --jar $ELASTICSEARCH_SQL_JAR \ --jar $SQL_TOOLBOX_JAR \ --environment $SQL_CONF \ --update "$SQL_STATEMENT_5" | grep "Job ID:" | sed 's/.* //g') diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka.sh new file mode 100755 index 0000000000000..0941cf200f1ec --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -Eeuo pipefail + +source "$(dirname "$0")"/test_sql_client_kafka_common.sh 2.0 2.0.1 5.0.0 5.0 "kafka" "universal" diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka010.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka010.sh new file mode 100755 index 0000000000000..66bef6623a156 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka010.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -Eeuo pipefail + +source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.10 0.10.2.0 3.2.0 3.2 "kafka-0.10" "0.10" diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka011.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka011.sh new file mode 100755 index 0000000000000..e8ab9d0b29d10 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka011.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -Eeuo pipefail + +source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.11 0.11.0.2 3.2.0 3.2 "kafka-0.11" "0.11" diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka_common.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka_common.sh new file mode 100755 index 0000000000000..08ed59b419c22 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka_common.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +KAFKA_CONNECTOR_VERSION="$1" +KAFKA_VERSION="$2" +CONFLUENT_VERSION="$3" +CONFLUENT_MAJOR_VERSION="$4" +KAFKA_SQL_JAR="$5" +KAFKA_SQL_VERSION="$6" + +source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/kafka_sql_common.sh \ + $KAFKA_CONNECTOR_VERSION \ + $KAFKA_VERSION \ + $CONFLUENT_VERSION \ + $CONFLUENT_MAJOR_VERSION \ + $KAFKA_SQL_VERSION + +################################################################################ +# Prepare connectors +################################################################################ + +function sql_cleanup() { + # don't call ourselves again for another signal interruption + trap "exit -1" INT + # don't call ourselves again for normal exit + trap "" EXIT + + stop_kafka_cluster +} +trap sql_cleanup INT +trap sql_cleanup EXIT + +# prepare Kafka +echo "Preparing Kafka $KAFKA_VERSION..." + +setup_kafka_dist + +start_kafka_cluster + +create_kafka_json_source test-json +create_kafka_topic 1 1 test-avro + +################################################################################ +# Prepare Flink +################################################################################ + +echo "Preparing Flink..." + +start_cluster +start_taskmanagers 2 + +################################################################################ +# Run SQL statements +################################################################################ + +echo "Testing SQL statements..." + +SQL_TOOLBOX_JAR=$END_TO_END_DIR/flink-sql-client-test/target/SqlToolbox.jar +SQL_JARS_DIR=$END_TO_END_DIR/flink-sql-client-test/target/sql-jars + +AVRO_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "avro" ) +JSON_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "json" ) +KAFKA_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "${KAFKA_SQL_JAR}_" ) + +# create session environment file +RESULT=$TEST_DATA_DIR/result +SQL_CONF=$TEST_DATA_DIR/sql-client-session.conf + +cat >> $SQL_CONF << EOF +tables: +EOF + +get_kafka_json_source_schema test-json JsonSourceTable >> $SQL_CONF + +cat >> $SQL_CONF << EOF + - name: AvroBothTable + type: source-sink-table + update-mode: append + schema: + - name: event_timestamp + type: VARCHAR + - name: user + type: VARCHAR + - name: message + type: VARCHAR + - name: duplicate_count + type: BIGINT + connector: + type: kafka + version: "$KAFKA_SQL_VERSION" + topic: test-avro + startup-mode: earliest-offset + properties: + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + format: + type: avro + avro-schema: > + { + "namespace": "org.apache.flink.table.tests", + "type": "record", + "name": "NormalizedEvent", + "fields": [ + {"name": "event_timestamp", "type": "string"}, + {"name": "user", "type": ["string", "null"]}, + {"name": "message", "type": "string"}, + {"name": "duplicate_count", "type": "long"} + ] + } + - name: CsvSinkTable + type: sink-table + update-mode: append + schema: + - name: event_timestamp + type: VARCHAR + - name: user + type: VARCHAR + - name: message + type: VARCHAR + - name: duplicate_count + type: BIGINT + - name: constant + type: VARCHAR + connector: + type: filesystem + path: $RESULT + format: + type: csv + fields: + - name: event_timestamp + type: VARCHAR + - name: user + type: VARCHAR + - name: message + type: VARCHAR + - name: duplicate_count + type: BIGINT + - name: constant + type: VARCHAR + +functions: + - name: RegReplace + from: class + class: org.apache.flink.table.toolbox.StringRegexReplaceFunction +EOF + +# submit SQL statements + +echo "Executing SQL: Kafka $KAFKA_VERSION JSON -> Kafka $KAFKA_VERSION Avro" + +SQL_STATEMENT_1=$(cat << EOF +INSERT INTO AvroBothTable + SELECT + CAST(TUMBLE_START(rowtime, INTERVAL '1' HOUR) AS VARCHAR) AS event_timestamp, + user, + RegReplace(event.message, ' is ', ' was ') AS message, + COUNT(*) AS duplicate_count + FROM JsonSourceTable + WHERE user IS NOT NULL + GROUP BY + user, + event.message, + TUMBLE(rowtime, INTERVAL '1' HOUR) +EOF +) + +echo "$SQL_STATEMENT_1" + +$FLINK_DIR/bin/sql-client.sh embedded \ + --jar $KAFKA_SQL_JAR \ + --jar $JSON_SQL_JAR \ + --jar $AVRO_SQL_JAR \ + --jar $SQL_TOOLBOX_JAR \ + --environment $SQL_CONF \ + --update "$SQL_STATEMENT_1" + +echo "Executing SQL: Kafka $KAFKA_VERSION Avro -> Filesystem CSV" + +SQL_STATEMENT_2=$(cat << EOF +INSERT INTO CsvSinkTable + SELECT AvroBothTable.*, RegReplace('Test constant folding.', 'Test', 'Success') AS constant + FROM AvroBothTable +EOF +) + +echo "$SQL_STATEMENT_2" + +$FLINK_DIR/bin/sql-client.sh embedded \ + --jar $KAFKA_SQL_JAR \ + --jar $JSON_SQL_JAR \ + --jar $AVRO_SQL_JAR \ + --jar $SQL_TOOLBOX_JAR \ + --environment $SQL_CONF \ + --update "$SQL_STATEMENT_2" + +echo "Waiting for CSV results..." +for i in {1..10}; do + if [ -e $RESULT ]; then + CSV_LINE_COUNT=`cat $RESULT | wc -l` + if [ $((CSV_LINE_COUNT)) -eq 4 ]; then + break + fi + fi + sleep 5 +done + +check_result_hash "SQL Client Kafka $KAFKA_VERSION" $RESULT "0a1bf8bf716069b7269f575f87a802c0" diff --git a/flink-end-to-end-tests/test-scripts/test_stream_state_ttl.sh b/flink-end-to-end-tests/test-scripts/test_stream_state_ttl.sh index fb911f327ff90..1a634260db630 100755 --- a/flink-end-to-end-tests/test-scripts/test_stream_state_ttl.sh +++ b/flink-end-to-end-tests/test-scripts/test_stream_state_ttl.sh @@ -22,9 +22,8 @@ source "$(dirname "$0")"/common.sh STATE_BACKEND_TYPE="${1:-file}" STATE_BACKEND_FILE_ASYNC="${2:-false}" TTL="${3:-1000}" -PRECISION="${4:-5}" -PARALLELISM="${5-3}" -UPDATE_NUM="${6-1000}" +PARALLELISM="${4-3}" +UPDATE_NUM="${5-1000}" CHECKPOINT_DIR="file://$TEST_DATA_DIR/savepoint-e2e-test-chckpt-dir" @@ -46,15 +45,19 @@ trap test_cleanup INT trap test_cleanup EXIT start_cluster -start_taskmanagers $PARALLELISM +if [ "${PARALLELISM}" -gt "1" ]; then + start_taskmanagers $(expr ${PARALLELISM} - 1) +fi function job_id() { - CMD="${FLINK_DIR}/bin/flink run -d -p ${PARALLELISM} ${TEST_PROGRAM_JAR} \ + if [ -n "$1" ]; then + SP="-s $1" + fi + CMD="${FLINK_DIR}/bin/flink run -d ${SP} -p ${PARALLELISM} ${TEST_PROGRAM_JAR} \ --test.semantics exactly-once \ --environment.parallelism ${PARALLELISM} \ --state_backend ${STATE_BACKEND_TYPE} \ --state_ttl_verifier.ttl_milli ${TTL} \ - --state_ttl_verifier.precision_milli ${PRECISION} \ --state_backend.checkpoint_directory ${CHECKPOINT_DIR} \ --state_backend.file.async ${STATE_BACKEND_FILE_ASYNC} \ --update_generator_source.sleep_time 10 \ @@ -73,7 +76,7 @@ SAVEPOINT_PATH=$(take_savepoint ${JOB} ${TEST_DATA_DIR} \ cancel_job ${JOB} -JOB_CMD=$(job_id) +JOB_CMD=$(job_id ${SAVEPOINT_PATH}) echo ${JOB_CMD} JOB=$(${JOB_CMD} | grep 'Job has been submitted with JobID' | sed 's/.* //g') wait_job_running ${JOB} diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh index 17389ad6f4ec2..2756197fdf6e6 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh @@ -17,27 +17,40 @@ # limitations under the License. ################################################################################ -source "$(dirname "$0")"/common.sh - -TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-streaming-file-sink-test/target/StreamingFileSinkProgram.jar" +OUT_TYPE="${1:-local}" # other type: s3 -OUTPUT_PATH="$TEST_DATA_DIR/out" - -function wait_for_restart { - local base_num_restarts=$1 - - local current_num_restarts=${base_num_restarts} - local expected_num_restarts=$((current_num_restarts + 1)) +source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/common_s3.sh + +s3_setup hadoop +set_conf_ssl "mutual" + +OUT=temp/test_streaming_file_sink-$(uuidgen) +OUTPUT_PATH="$TEST_DATA_DIR/$OUT" +S3_OUTPUT_PATH="s3://$IT_CASE_S3_BUCKET/$OUT" + +mkdir -p $OUTPUT_PATH + +if [ "${OUT_TYPE}" == "local" ]; then + echo "Use local output" + JOB_OUTPUT_PATH=${OUTPUT_PATH} +elif [ "${OUT_TYPE}" == "s3" ]; then + echo "Use s3 output" + JOB_OUTPUT_PATH=${S3_OUTPUT_PATH} +else + echo "Unknown output type: ${OUT_TYPE}" + exit 1 +fi - echo "Waiting for restart to happen" - while ! [[ ${current_num_restarts} -eq ${expected_num_restarts} ]]; do - sleep 5 - current_num_restarts=$(get_job_metric ${JOB_ID} "fullRestarts") - if [[ -z ${current_num_restarts} ]]; then - current_num_restarts=${base_num_restarts} - fi - done +# make sure we delete the file at the end +function out_cleanup { + s3_delete_by_full_path_prefix $OUT } +if [ "${OUT_TYPE}" == "s3" ]; then + trap out_cleanup EXIT +fi + +TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-streaming-file-sink-test/target/StreamingFileSinkProgram.jar" ################################### # Get all lines in part files and sort them numerically. @@ -47,10 +60,32 @@ function wait_for_restart { # Arguments: # None # Returns: -# None +# sorted content of part files ################################### function get_complete_result { - find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g + if [ "${OUT_TYPE}" == "s3" ]; then + rm -rf $OUTPUT_PATH; mkdir -p $OUTPUT_PATH + s3_get_by_full_path_and_filename_prefix ${TEST_DATA_DIR} "${OUT}" "part-" + fi + find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g +} + +################################### +# Get total number of lines in part files. +# +# Globals: +# OUT +# Arguments: +# None +# Returns: +# line number in part files +################################### +function get_total_number_of_valid_lines { + if [ "${OUT_TYPE}" == "local" ]; then + get_complete_result | wc -l | tr -d '[:space:]' + elif [ "${OUT_TYPE}" == "s3" ]; then + s3_get_number_of_lines_by_prefix "${OUT}" "part-" + fi } ################################### @@ -83,7 +118,7 @@ function wait_for_complete_result { sleep ${polling_interval} ((seconds_elapsed += ${polling_interval})) - number_of_values=$(get_complete_result | wc -l | tr -d '[:space:]') + number_of_values=$(get_total_number_of_valid_lines) if [[ ${previous_number_of_values} -ne ${number_of_values} ]]; then echo "Number of produced values ${number_of_values}/${expected_number_of_values}" previous_number_of_values=${number_of_values} @@ -98,7 +133,7 @@ start_cluster "${FLINK_DIR}/bin/taskmanager.sh" start echo "Submitting job." -CLIENT_OUTPUT=$("$FLINK_DIR/bin/flink" run -d "${TEST_PROGRAM_JAR}" --outputPath "${OUTPUT_PATH}") +CLIENT_OUTPUT=$("$FLINK_DIR/bin/flink" run -d "${TEST_PROGRAM_JAR}" --outputPath "${JOB_OUTPUT_PATH}") JOB_ID=$(echo "${CLIENT_OUTPUT}" | grep "Job has been submitted with JobID" | sed 's/.* //g') if [[ -z $JOB_ID ]]; then @@ -117,7 +152,7 @@ kill_random_taskmanager echo "Starting TM" "$FLINK_DIR/bin/taskmanager.sh" start -wait_for_restart 0 +wait_for_restart_to_complete 0 ${JOB_ID} echo "Killing 2 TMs" kill_random_taskmanager @@ -127,7 +162,7 @@ echo "Starting 2 TMs" "$FLINK_DIR/bin/taskmanager.sh" start "$FLINK_DIR/bin/taskmanager.sh" start -wait_for_restart 1 +wait_for_restart_to_complete 1 ${JOB_ID} echo "Waiting until all values have been produced" wait_for_complete_result 60000 300 diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh new file mode 100755 index 0000000000000..045bb0933f8e4 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -Eeuo pipefail + +source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/kafka-common.sh 2.0.1 5.0.0 5.0 + +source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka-test/target/KafkaExample.jar diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh index c9cc19db71d55..97a71f9005262 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh @@ -17,97 +17,10 @@ # limitations under the License. ################################################################################ -source "$(dirname "$0")"/common.sh -source "$(dirname "$0")"/kafka-common.sh - -setup_kafka_dist -start_kafka_cluster - -# modify configuration to have enough slots -cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak -sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: 3/" $FLINK_DIR/conf/flink-conf.yaml - -start_cluster - -function test_cleanup { - # don't call ourselves again for another signal interruption - trap "exit -1" INT - # don't call ourselves again for normal exit - trap "" EXIT - - stop_kafka_cluster - - # revert our modifications to the Flink distribution - mv -f $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml -} -trap test_cleanup INT -trap test_cleanup EXIT - -# create the required topics -create_kafka_topic 1 1 test-input -create_kafka_topic 1 1 test-output - -# run the Flink job (detached mode) -$FLINK_DIR/bin/flink run -d $FLINK_DIR/examples/streaming/Kafka010Example.jar \ - --input-topic test-input --output-topic test-output \ - --prefix=PREFIX \ - --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest \ - --flink.partition-discovery.interval-millis 1000 - -function verify_output { - local expected=$(printf $1) +set -Eeuo pipefail - if [[ "$2" != "$expected" ]]; then - echo "Output from Flink program does not match expected output." - echo -e "EXPECTED FOR KEY: --$expected--" - echo -e "ACTUAL: --$2--" - exit 1 - fi -} - -echo "Sending messages to Kafka topic [test-input] ..." -# send some data to Kafka -send_messages_to_kafka "elephant,5,45218\nsquirrel,12,46213\nbee,3,51348\nsquirrel,22,52444\nbee,10,53412\nelephant,9,54867" test-input - -echo "Verifying messages from Kafka topic [test-output] ..." - -KEY_1_MSGS=$(read_messages_from_kafka 6 test-output elephant_consumer | grep elephant) -KEY_2_MSGS=$(read_messages_from_kafka 6 test-output squirrel_consumer | grep squirrel) -KEY_3_MSGS=$(read_messages_from_kafka 6 test-output bee_consumer | grep bee) - -# check all keys; make sure we have actual newlines in the string, not "\n" -verify_output "elephant,5,45218\nelephant,14,54867" "$KEY_1_MSGS" -verify_output "squirrel,12,46213\nsquirrel,34,52444" "$KEY_2_MSGS" -verify_output "bee,3,51348\nbee,13,53412" "$KEY_3_MSGS" - -# now, we add a new partition to the topic -echo "Repartitioning Kafka topic [test-input] ..." -modify_num_partitions test-input 2 - -if (( $(get_num_partitions test-input) != 2 )); then - echo "Failed adding a partition to test-input topic." - exit 1 -fi - -# send some more messages to Kafka -echo "Sending more messages to Kafka topic [test-input] ..." -send_messages_to_kafka "elephant,13,64213\ngiraffe,9,65555\nbee,5,65647\nsquirrel,18,66413" test-input - -# verify that our assumption that the new partition actually has written messages is correct -if (( $(get_partition_end_offset test-input 1) == 0 )); then - echo "The newly created partition does not have any new messages, and therefore partition discovery cannot be verified." - exit 1 -fi - -# all new messages should have been consumed, and has produced correct output -echo "Verifying messages from Kafka topic [test-output] ..." +source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/kafka-common.sh 0.10.2.0 3.2.0 3.2 -KEY_1_MSGS=$(read_messages_from_kafka 4 test-output elephant_consumer | grep elephant) -KEY_2_MSGS=$(read_messages_from_kafka 4 test-output squirrel_consumer | grep squirrel) -KEY_3_MSGS=$(read_messages_from_kafka 4 test-output bee_consumer | grep bee) -KEY_4_MSGS=$(read_messages_from_kafka 10 test-output giraffe_consumer | grep giraffe) +source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka010-test/target/Kafka010Example.jar -verify_output "elephant,27,64213" "$KEY_1_MSGS" -verify_output "squirrel,52,66413" "$KEY_2_MSGS" -verify_output "bee,18,65647" "$KEY_3_MSGS" -verify_output "giraffe,9,65555" "$KEY_4_MSGS" diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka011.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka011.sh new file mode 100755 index 0000000000000..84fa6edc90eca --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_streaming_kafka011.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh +source "$(dirname "$0")"/kafka-common.sh 0.11.0.2 3.2.0 3.2 + +source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka011-test/target/Kafka011Example.jar + diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka_common.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka_common.sh new file mode 100644 index 0000000000000..0583638b3bcf9 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_streaming_kafka_common.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +KAFKA_EXAMPLE_JAR="$1" + +setup_kafka_dist +start_kafka_cluster + +# modify configuration to have enough slots +cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak +sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: 3/" $FLINK_DIR/conf/flink-conf.yaml + +start_cluster + +function test_cleanup { + # don't call ourselves again for another signal interruption + trap "exit -1" INT + # don't call ourselves again for normal exit + trap "" EXIT + + stop_kafka_cluster + + # revert our modifications to the Flink distribution + mv -f $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml +} +trap test_cleanup INT +trap test_cleanup EXIT + +# create the required topics +create_kafka_topic 1 1 test-input +create_kafka_topic 1 1 test-output + +# run the Flink job (detached mode) +$FLINK_DIR/bin/flink run -d $KAFKA_EXAMPLE_JAR \ + --input-topic test-input --output-topic test-output \ + --prefix=PREFIX \ + --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest \ + --transaction.timeout.ms 900000 \ + --flink.partition-discovery.interval-millis 1000 + +function verify_output { + local expected=$(printf $1) + + if [[ "$2" != "$expected" ]]; then + echo "Output from Flink program does not match expected output." + echo -e "EXPECTED FOR KEY: --$expected--" + echo -e "ACTUAL: --$2--" + exit 1 + fi +} + +echo "Sending messages to Kafka topic [test-input] ..." +# send some data to Kafka +send_messages_to_kafka "elephant,5,45218\nsquirrel,12,46213\nbee,3,51348\nsquirrel,22,52444\nbee,10,53412\nelephant,9,54867" test-input + +echo "Verifying messages from Kafka topic [test-output] ..." + +KEY_1_MSGS=$(read_messages_from_kafka 6 test-output elephant_consumer | grep elephant) +KEY_2_MSGS=$(read_messages_from_kafka 6 test-output squirrel_consumer | grep squirrel) +KEY_3_MSGS=$(read_messages_from_kafka 6 test-output bee_consumer | grep bee) + +# check all keys; make sure we have actual newlines in the string, not "\n" +verify_output "elephant,5,45218\nelephant,14,54867" "$KEY_1_MSGS" +verify_output "squirrel,12,46213\nsquirrel,34,52444" "$KEY_2_MSGS" +verify_output "bee,3,51348\nbee,13,53412" "$KEY_3_MSGS" + +# now, we add a new partition to the topic +echo "Repartitioning Kafka topic [test-input] ..." +modify_num_partitions test-input 2 + +if (( $(get_num_partitions test-input) != 2 )); then + echo "Failed adding a partition to test-input topic." + exit 1 +fi + +# send some more messages to Kafka +echo "Sending more messages to Kafka topic [test-input] ..." +send_messages_to_kafka "elephant,13,64213\ngiraffe,9,65555\nbee,5,65647\nsquirrel,18,66413" test-input + +# verify that our assumption that the new partition actually has written messages is correct +if (( $(get_partition_end_offset test-input 1) == 0 )); then + echo "The newly created partition does not have any new messages, and therefore partition discovery cannot be verified." + exit 1 +fi + +# all new messages should have been consumed, and has produced correct output +echo "Verifying messages from Kafka topic [test-output] ..." + +KEY_1_MSGS=$(read_messages_from_kafka 4 test-output elephant_consumer | grep elephant) +KEY_2_MSGS=$(read_messages_from_kafka 4 test-output squirrel_consumer | grep squirrel) +KEY_3_MSGS=$(read_messages_from_kafka 4 test-output bee_consumer | grep bee) +KEY_4_MSGS=$(read_messages_from_kafka 10 test-output giraffe_consumer | grep giraffe) + +verify_output "elephant,27,64213" "$KEY_1_MSGS" +verify_output "squirrel,52,66413" "$KEY_2_MSGS" +verify_output "bee,18,65647" "$KEY_3_MSGS" +verify_output "giraffe,9,65555" "$KEY_4_MSGS" diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh index c9ef15d3dc533..5f2dea2ea6a01 100755 --- a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh +++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh @@ -24,7 +24,8 @@ FLINK_TARBALL_DIR=$TEST_DATA_DIR FLINK_TARBALL=flink.tar.gz FLINK_DIRNAME=$(basename $FLINK_DIR) -MAX_RETRY_SECONDS=800 +MAX_RETRY_SECONDS=120 +CLUSTER_SETUP_RETRIES=3 echo "Flink Tarball directory $FLINK_TARBALL_DIR" echo "Flink tarball filename $FLINK_TARBALL" @@ -33,20 +34,6 @@ echo "End-to-end directory $END_TO_END_DIR" docker --version docker-compose --version -mkdir -p $FLINK_TARBALL_DIR -tar czf $FLINK_TARBALL_DIR/$FLINK_TARBALL -C $(dirname $FLINK_DIR) . - -echo "Building Hadoop Docker container" -until docker build --build-arg HADOOP_VERSION=2.8.4 -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/Dockerfile -t flink/docker-hadoop-secure-cluster:latest $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/; do - # with all the downloading and ubuntu updating a lot of flakiness can happen, make sure - # we don't immediately fail - echo "Something went wrong while building the Docker image, retrying ..." - sleep 2 -done - -echo "Starting Hadoop cluster" -docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml up -d - # make sure we stop our cluster at the end function cluster_shutdown { # don't call ourselves again for another signal interruption @@ -60,12 +47,71 @@ function cluster_shutdown { trap cluster_shutdown INT trap cluster_shutdown EXIT -until docker cp $FLINK_TARBALL_DIR/$FLINK_TARBALL master:/home/hadoop-user/; do - # we're retrying this one because we don't know yet if the container is ready - echo "Uploading Flink tarball to docker master failed, retrying ..." - sleep 5 +function start_hadoop_cluster() { + echo "Starting Hadoop cluster" + docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml up -d + + # wait for kerberos to be set up + start_time=$(date +%s) + until docker logs master 2>&1 | grep -q "Finished master initialization"; do + current_time=$(date +%s) + time_diff=$((current_time - start_time)) + + if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then + return 1 + else + echo "Waiting for hadoop cluster to come up. We have been trying for $time_diff seconds, retrying ..." + sleep 10 + fi + done + + # perform health checks + if ! { [ $(docker inspect -f '{{.State.Running}}' master 2>&1) = 'true' ] && + [ $(docker inspect -f '{{.State.Running}}' slave1 2>&1) = 'true' ] && + [ $(docker inspect -f '{{.State.Running}}' slave2 2>&1) = 'true' ] && + [ $(docker inspect -f '{{.State.Running}}' kdc 2>&1) = 'true' ]; }; + then + return 1 + fi + + return 0 +} + +mkdir -p $FLINK_TARBALL_DIR +tar czf $FLINK_TARBALL_DIR/$FLINK_TARBALL -C $(dirname $FLINK_DIR) . + +echo "Building Hadoop Docker container" +until docker build --build-arg HADOOP_VERSION=2.8.4 \ + -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/Dockerfile \ + -t flink/docker-hadoop-secure-cluster:latest \ + $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/; +do + # with all the downloading and ubuntu updating a lot of flakiness can happen, make sure + # we don't immediately fail + echo "Something went wrong while building the Docker image, retrying ..." + sleep 2 +done + +CLUSTER_STARTED=1 +for (( i = 0; i < $CLUSTER_SETUP_RETRIES; i++ )) +do + if start_hadoop_cluster; then + echo "Cluster started successfully." + CLUSTER_STARTED=0 + break #continue test, cluster set up succeeded + fi + + echo "ERROR: Could not start hadoop cluster. Retrying..." + docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml down done +if [[ ${CLUSTER_STARTED} -ne 0 ]]; then + echo "ERROR: Could not start hadoop cluster. Aborting..." + exit 1 +fi + +docker cp $FLINK_TARBALL_DIR/$FLINK_TARBALL master:/home/hadoop-user/ + # now, at least the container is ready docker exec -it master bash -c "tar xzf /home/hadoop-user/$FLINK_TARBALL --directory /home/hadoop-user/" @@ -73,6 +119,7 @@ docker exec -it master bash -c "tar xzf /home/hadoop-user/$FLINK_TARBALL --direc docker exec -it master bash -c "echo \"security.kerberos.login.keytab: /home/hadoop-user/hadoop-user.keytab\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" docker exec -it master bash -c "echo \"security.kerberos.login.principal: hadoop-user\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" docker exec -it master bash -c "echo \"slot.request.timeout: 60000\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" +docker exec -it master bash -c "echo \"containerized.heap-cutoff-min: 100\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" echo "Flink config:" docker exec -it master bash -c "cat /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" @@ -84,33 +131,28 @@ OUTPUT_PATH=hdfs:///user/hadoop-user/wc-out-$RANDOM start_time=$(date +%s) # it's important to run this with higher parallelism, otherwise we might risk that # JM and TM are on the same YARN node and that we therefore don't test the keytab shipping -until docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 2000 -yjm 2000 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH"; do - current_time=$(date +%s) - time_diff=$((current_time - start_time)) - - if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then - echo "We tried running the job for $time_diff seconds, max is $MAX_RETRY_SECONDS seconds, aborting" - mkdir -p $TEST_DATA_DIR/logs - echo "Hadoop logs:" - docker cp master:/var/log/hadoop/* $TEST_DATA_DIR/logs/ - for f in $TEST_DATA_DIR/logs/*; do - echo "$f:" - cat $f - done - echo "Docker logs:" - docker logs master - exit 1 - else - echo "Running the Flink job failed, might be that the cluster is not ready yet. We have been trying for $time_diff seconds, retrying ..." - sleep 5 - fi -done - -docker exec -it master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user" -docker exec -it master bash -c "hdfs dfs -ls $OUTPUT_PATH" -OUTPUT=$(docker exec -it master bash -c "hdfs dfs -cat $OUTPUT_PATH/*") -docker exec -it master bash -c "kdestroy" -echo "$OUTPUT" +if docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && \ + /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1000 -yjm 1000 \ + -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH"; +then + docker exec -it master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user" + docker exec -it master bash -c "hdfs dfs -ls $OUTPUT_PATH" + OUTPUT=$(docker exec -it master bash -c "hdfs dfs -cat $OUTPUT_PATH/*") + docker exec -it master bash -c "kdestroy" + echo "$OUTPUT" +else + echo "Running the job failed." + mkdir -p $TEST_DATA_DIR/logs + echo "Hadoop logs:" + docker cp master:/var/log/hadoop/* $TEST_DATA_DIR/logs/ + for f in $TEST_DATA_DIR/logs/*; do + echo "$f:" + cat $f + done + echo "Docker logs:" + docker logs master + exit 1 +fi if [[ ! "$OUTPUT" =~ "consummation,1" ]]; then echo "Output does not contain (consummation, 1) as required" @@ -139,7 +181,10 @@ fi echo "Running Job without configured keytab, the exception you see below is expected" docker exec -it master bash -c "echo \"\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml" # verify that it doesn't work if we don't configure a keytab -OUTPUT=$(docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH") +OUTPUT=$(docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && \ + /home/hadoop-user/$FLINK_DIRNAME/bin/flink run \ + -m yarn-cluster -yn 3 -ys 1 -ytm 1000 -yjm 1000 -p 3 \ + /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH") echo "$OUTPUT" if [[ ! "$OUTPUT" =~ "Hadoop security with Kerberos is enabled but the login user does not have Kerberos credentials" ]]; then diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml new file mode 100644 index 0000000000000..c120db2276f6c --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml @@ -0,0 +1,106 @@ + + + + 4.0.0 + + + flink-examples-build-helper + org.apache.flink + 1.7-SNAPSHOT + .. + + + flink-examples-streaming-state-machine_${scala.binary.version} + flink-examples-streaming-state-machine + jar + + + + org.apache.flink + flink-examples-streaming_${scala.binary.version} + ${project.version} + + + + + StateMachineExample + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + false + false + + + org.apache.flink.streaming.examples.statemachine.StateMachineExample + + + + + org.apache.flink:flink-connector-kafka* + org.apache.flink:flink-examples-streaming* + org.apache.kafka:* + + + + + + org.apache.flink:flink-examples-streaming_* + + org/apache/flink/streaming/examples/statemachine/** + + + + org.apache.kafka:* + + LICENSE + + NOTICE + + + + + + + + + + + diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..7a2f9e33340c9 --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-examples-streaming-state-machine +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml b/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml new file mode 100644 index 0000000000000..74c5b3bd1055e --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + + flink-examples-build-helper + org.apache.flink + 1.7-SNAPSHOT + .. + + + flink-examples-streaming-twitter_${scala.binary.version} + flink-examples-streaming-twitter + jar + + + + org.apache.flink + flink-examples-streaming_${scala.binary.version} + ${project.version} + + + + + Twitter + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + false + false + + + org.apache.flink.streaming.examples.twitter.TwitterExample + + + + + org.apache.flink:flink-connector-twitter* + org.apache.flink:flink-examples-streaming* + + + + + org.apache.flink:*flink-examples-streaming_* + + org/apache/flink/streaming/examples/twitter/** + + + + + + + + + + + diff --git a/flink-examples/flink-examples-build-helper/pom.xml b/flink-examples/flink-examples-build-helper/pom.xml new file mode 100644 index 0000000000000..30c6b2397856c --- /dev/null +++ b/flink-examples/flink-examples-build-helper/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + + + flink-examples + org.apache.flink + 1.7-SNAPSHOT + .. + + + flink-examples-build-helper + pom + This is a utility module for building example jars to be used in flink-dist. + + + flink-examples-streaming-twitter + flink-examples-streaming-state-machine + + + diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml index cbeef2a10cd04..ecead55fad0d4 100644 --- a/flink-examples/flink-examples-streaming/pom.xml +++ b/flink-examples/flink-examples-streaming/pom.xml @@ -491,117 +491,6 @@ under the License. - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - - state-machine-example - package - - shade - - - false - false - false - - - org.apache.flink.streaming.examples.statemachine.StateMachineExample - - - StateMachineExample - - - * - - org/apache/flink/streaming/examples/statemachine/** - org/apache/flink/streaming/util/serialization/** - org/apache/flink/streaming/connectors/kafka/** - org/apache/kafka/** - kafka/ - - - - - - - - fat-jar-kafka-010-example - package - - shade - - - false - false - false - - - org.apache.flink.streaming.examples.kafka.Kafka010Example - - - Kafka010Example - - - - * - - org/apache/flink/streaming/examples/kafka/** - org/apache/flink/streaming/** - org/apache/kafka/** - org/apache/curator/** - org/apache/zookeeper/** - org/apache/jute/** - org/I0Itec/** - jline/** - com/yammer/** - kafka/** - - - - - - - - fat-jar-twitter-example - package - - shade - - - false - false - false - - - org.apache.flink.streaming.examples.twitter.TwitterExample - - - Twitter - - - * - - org/apache/flink/twitter/shaded/com/google/common/** - org/apache/flink/streaming/examples/twitter/** - org/apache/flink/streaming/connectors/twitter/** - org/apache/http/** - com/twitter/** - build.properties - - - - - - - - diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java index d123615c7fca2..13db58ae187b1 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java @@ -115,7 +115,7 @@ public static void main(String[] args) throws Exception { // ************************************************************************* /** - * Generate BOUND number of random integer pairs from the range from 0 to BOUND/2. + * Generate BOUND number of random integer pairs from the range from 1 to BOUND/2. */ private static class RandomFibonacciSource implements SourceFunction> { private static final long serialVersionUID = 1L; diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java deleted file mode 100644 index 62bfd4fc6c5b4..0000000000000 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.examples.kafka; - -import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.state.ValueState; -import org.apache.flink.api.common.state.ValueStateDescriptor; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; -import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; -import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; -import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; - -import javax.annotation.Nullable; - -/** - * A simple example that shows how to read from and write to Kafka. This will read String messages - * from the input topic, parse them into a POJO type {@link KafkaEvent}, group by some key, and finally - * perform a rolling addition on each key for which the results are written back to another topic. - * - *

    This example also demonstrates using a watermark assigner to generate per-partition - * watermarks directly in the Flink Kafka consumer. For demonstration purposes, it is assumed that - * the String messages are of formatted as a (word,frequency,timestamp) tuple. - * - *

    Example usage: - * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer - */ -public class Kafka010Example { - - public static void main(String[] args) throws Exception { - // parse input arguments - final ParameterTool parameterTool = ParameterTool.fromArgs(args); - - if (parameterTool.getNumberOfParameters() < 5) { - System.out.println("Missing parameters!\n" + - "Usage: Kafka --input-topic --output-topic " + - "--bootstrap.servers " + - "--zookeeper.connect --group.id "); - return; - } - - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.getConfig().disableSysoutLogging(); - env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)); - env.enableCheckpointing(5000); // create a checkpoint every 5 seconds - env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface - env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - - DataStream input = env - .addSource( - new FlinkKafkaConsumer010<>( - parameterTool.getRequired("input-topic"), - new KafkaEventSchema(), - parameterTool.getProperties()) - .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) - .keyBy("word") - .map(new RollingAdditionMapper()); - - input.addSink( - new FlinkKafkaProducer010<>( - parameterTool.getRequired("output-topic"), - new KafkaEventSchema(), - parameterTool.getProperties())); - - env.execute("Kafka 0.10 Example"); - } - - /** - * A {@link RichMapFunction} that continuously outputs the current total frequency count of a key. - * The current total count is keyed state managed by Flink. - */ - private static class RollingAdditionMapper extends RichMapFunction { - - private static final long serialVersionUID = 1180234853172462378L; - - private transient ValueState currentTotalCount; - - @Override - public KafkaEvent map(KafkaEvent event) throws Exception { - Integer totalCount = currentTotalCount.value(); - - if (totalCount == null) { - totalCount = 0; - } - totalCount += event.getFrequency(); - - currentTotalCount.update(totalCount); - - return new KafkaEvent(event.getWord(), totalCount, event.getTimestamp()); - } - - @Override - public void open(Configuration parameters) throws Exception { - currentTotalCount = getRuntimeContext().getState(new ValueStateDescriptor<>("currentTotalCount", Integer.class)); - } - } - - /** - * A custom {@link AssignerWithPeriodicWatermarks}, that simply assumes that the input stream - * records are strictly ascending. - * - *

    Flink also ships some built-in convenience assigners, such as the - * {@link BoundedOutOfOrdernessTimestampExtractor} and {@link AscendingTimestampExtractor} - */ - private static class CustomWatermarkExtractor implements AssignerWithPeriodicWatermarks { - - private static final long serialVersionUID = -742759155861320823L; - - private long currentTimestamp = Long.MIN_VALUE; - - @Override - public long extractTimestamp(KafkaEvent event, long previousElementTimestamp) { - // the inputs are assumed to be of format (message,timestamp) - this.currentTimestamp = event.getTimestamp(); - return event.getTimestamp(); - } - - @Nullable - @Override - public Watermark getCurrentWatermark() { - return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); - } - } -} diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala deleted file mode 100644 index 9f4fdc4c29489..0000000000000 --- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.scala.examples.kafka - -import org.apache.flink.api.common.restartstrategy.RestartStrategies -import org.apache.flink.api.common.serialization.SimpleStringSchema -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala._ -import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer010, FlinkKafkaProducer010} - -/** - * An example that shows how to read from and write to Kafka. This will read String messages - * from the input topic, prefix them by a configured prefix and output to the output topic. - * - * Please pass the following arguments to run the example: - * {{{ - * --input-topic test-input - * --output-topic test-output - * --bootstrap.servers localhost:9092 - * --zookeeper.connect localhost:2181 - * --group.id myconsumer - * }}} - */ -object Kafka010Example { - - def main(args: Array[String]): Unit = { - - // parse input arguments - val params = ParameterTool.fromArgs(args) - - if (params.getNumberOfParameters < 4) { - println("Missing parameters!\n" - + "Usage: Kafka --input-topic --output-topic " - + "--bootstrap.servers " - + "--zookeeper.connect --group.id [--prefix ]") - return - } - - val prefix = params.get("prefix", "PREFIX:") - - - val env = StreamExecutionEnvironment.getExecutionEnvironment - env.getConfig.disableSysoutLogging - env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)) - // create a checkpoint every 5 seconds - env.enableCheckpointing(5000) - // make parameters available in the web interface - env.getConfig.setGlobalJobParameters(params) - - // create a Kafka streaming source consumer for Kafka 0.10.x - val kafkaConsumer = new FlinkKafkaConsumer010( - params.getRequired("input-topic"), - new SimpleStringSchema, - params.getProperties) - - val messageStream = env - .addSource(kafkaConsumer) - .map(in => prefix + in) - - // create a Kafka producer for Kafka 0.10.x - val kafkaProducer = new FlinkKafkaProducer010( - params.getRequired("output-topic"), - new SimpleStringSchema, - params.getProperties) - - // write data into Kafka - messageStream.addSink(kafkaProducer) - - env.execute("Kafka 0.10 Example") - } - -} diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml index 6a9b7e4379315..aa3a66f12f437 100644 --- a/flink-examples/pom.xml +++ b/flink-examples/pom.xml @@ -36,6 +36,7 @@ under the License. flink-examples-batch flink-examples-streaming flink-examples-table + flink-examples-build-helper diff --git a/flink-filesystems/flink-fs-hadoop-shaded/pom.xml b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml index b8029a187f21a..e5411f1c32451 100644 --- a/flink-filesystems/flink-fs-hadoop-shaded/pom.xml +++ b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml @@ -250,6 +250,7 @@ under the License. PropertyList-1.0.dtd META-INF/maven/** META-INF/services/javax.xml.stream.* + META-INF/LICENSE.txt diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..e4774491b4df6 --- /dev/null +++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE @@ -0,0 +1,2577 @@ +flink-fs-hadoop-shaded +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-annotations:3.1.0 +- org.apache.hadoop:hadoop-auth:3.1.0 +- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.3.2 +- commons-lang:commons-lang:2.6 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.3 +- com.google.guava:guava:11.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.7.0 +- com.fasterxml.jackson.core:jackson-core:2.7.8 +- com.fasterxml.jackson.core:jackson-databind:2.7.8 +- com.fasterxml.woodstox:woodstox-core:5.0.3 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +See bundled license files for details. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +See bundled license files for details. + +- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) + +This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of the Massachusetts Institute of Technology nor + the names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Other portions are under the same license from Intel: +http://sourceforge.net/projects/slicing-by-8/ +/*++ + * + * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + * + * This software program is licensed subject to the BSD License, + * available at http://www.opensource.org/licenses/bsd-license.html + * + * Abstract: The main routine + * + --*/ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +For hadoop-common-project/hadoop-common/src/main/native/gtest +--------------------------------------------------------------------- +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +re2j 1.1 +--------------------------------------------------------------------- +(GO license) +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +The binary distribution of this product bundles these dependencies under the +following license: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +bootstrap v3.3.6 +broccoli-asset-rev v2.4.2 +broccoli-funnel v1.0.1 +datatables v1.10.8 +em-helpers v0.5.13 +em-table v0.1.6 +ember v2.2.0 +ember-array-contains-helper v1.0.2 +ember-bootstrap v0.5.1 +ember-cli v1.13.13 +ember-cli-app-version v1.0.0 +ember-cli-babel v5.1.6 +ember-cli-content-security-policy v0.4.0 +ember-cli-dependency-checker v1.2.0 +ember-cli-htmlbars v1.0.2 +ember-cli-htmlbars-inline-precompile v0.3.1 +ember-cli-ic-ajax v0.2.1 +ember-cli-inject-live-reload v1.4.0 +ember-cli-jquery-ui v0.0.20 +ember-cli-qunit v1.2.1 +ember-cli-release v0.2.8 +ember-cli-shims v0.0.6 +ember-cli-sri v1.2.1 +ember-cli-test-loader v0.2.1 +ember-cli-uglify v1.2.0 +ember-d3 v0.1.0 +ember-data v2.1.0 +ember-disable-proxy-controllers v1.0.1 +ember-export-application-global v1.0.5 +ember-load-initializers v0.1.7 +ember-qunit v0.4.16 +ember-qunit-notifications v0.1.0 +ember-resolver v2.0.3 +ember-spin-spinner v0.2.3 +ember-truth-helpers v1.2.0 +jquery v2.1.4 +jquery-ui v1.11.4 +loader.js v3.3.0 +momentjs v2.10.6 +qunit v1.19.0 +select2 v4.0.0 +snippet-ss v1.11.0 +spin.js v2.3.2 +Azure Data Lake Store - Java client SDK 2.0.11 +JCodings 1.0.8 +Joni 2.1.2 +Mockito 1.8.5 +JUL to SLF4J bridge 1.7.25 +SLF4J API Module 1.7.25 +SLF4J LOG4J-12 Binding 1.7.25 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 +Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2003-2017 Optimatika + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For: +oj! Algorithms - version 43.0 +-------------------------------------------------------------------------------- + +Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +==== + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.3.4 +-------------------------------------------------------------------------------- +(HSQL License) +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +Java Servlet API 3.1.0 +servlet-api 2.5 +jsp-api 2.1 +jsr311-api 1.1.1 +Glassfish Jasper 6.1.14 +Servlet Specification 2.5 API 6.1.14 +-------------------------------------------------------------------------------- +(CDDL 1.0) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +jersey-client 1.19 +jersey-core 1.19 +jersey-grizzly2 1.19 +jersey-grizzly2-servlet 1.19 +jersey-json 1.19 +jersey-server 1.19 +jersey-servlet 1.19 +jersey-guice 1.19 +Jersey Test Framework - Grizzly 2 Module 1.19 +JAXB RI 2.2.3 +Java Architecture for XML Binding 2.2.11 +grizzly-framework 2.2.21 +grizzly-http 2.2.21 +grizzly-http-server 2.2.21 +grizzly-http-servlet 2.2.21 +grizzly-rcm 2.2.21 +-------------------------------------------------------------------------------- +(CDDL 1.1) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2. License Grants. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +Eclipse JDT Core 3.1.1 +-------------------------------------------------------------------------------- +(EPL v1.0) +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +ASM Core 5.0.4 +ASM Commons 5.0.2 +ASM Tree 5.0.2 +-------------------------------------------------------------------------------- +(3-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. +-------------------------------------------------------------------------------- +(2-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +"Java Concurrency in Practice" book annotations 1.0 +-------------------------------------------------------------------------------- +(CCAL v2.5) +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +The binary distribution of this product bundles these dependencies under the +following license: +jamon-runtime 2.4.1 +-------------------------------------------------------------------------------- +(MPL 2.0) + Mozilla Public License + Version 2.0 + +1. Definitions + +1.1. “Contributor” +means each individual or legal entity that creates, contributes to the creation +of, or owns Covered Software. + +1.2. “Contributor Version” +means the combination of the Contributions of others (if any) used by a +Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” +means Covered Software of a particular Contributor. + +1.4. “Covered Software” +means Source Code Form to which the initial Contributor has attached the notice +in Exhibit A, the Executable Form of such Source Code Form, and Modifications of +such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” +means + +that the initial Contributor has attached the notice described in Exhibit B to +the Covered Software; or + +that the Covered Software was made available under the terms of version 1.1 or +earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” +means any form of the work other than Source Code Form. + +1.7. “Larger Work” +means a work that combines Covered Software with other material, in a separate +file or files, that is not Covered Software. + +1.8. “License” +means this document. + +1.9. “Licensable” +means having the right to grant, to the maximum extent possible, whether at the +time of the initial grant or subsequently, any and all of the rights conveyed by +this License. + +1.10. “Modifications” +means any of the following: + +any file in Source Code Form that results from an addition to, deletion from, or +modification of the contents of Covered Software; or + +any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor +means any patent claim(s), including without limitation, method, process, and +apparatus claims, in any patent Licensable by such Contributor that would be +infringed, but for the grant of the License, by the making, using, selling, +offering for sale, having made, import, or transfer of either its Contributions +or its Contributor Version. + +1.12. “Secondary License” +means either the GNU General Public License, Version 2.0, the GNU Lesser General +Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, +or any later versions of those licenses. + +1.13. “Source Code Form” +means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) +means an individual or a legal entity exercising rights under this License. For +legal entities, “You” includes any entity that controls, is controlled by, +or is under common control with You. For purposes of this definition, +“control” means (a) the power, direct or indirect, to cause the direction or +management of such entity, whether by contract or otherwise, or (b) ownership of +more than fifty percent (50%) of the outstanding shares or beneficial ownership +of such entity. + +2. License Grants and Conditions + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license: + +under intellectual property rights (other than patent or trademark) Licensable +by such Contributor to use, reproduce, make available, modify, display, perform, +distribute, and otherwise exploit its Contributions, either on an unmodified +basis, with Modifications, or as part of a Larger Work; and + +under Patent Claims of such Contributor to make, use, sell, offer for sale, have +made, import, and otherwise transfer either its Contributions or its Contributor +Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution become +effective for each Contribution on the date the Contributor first distributes +such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under this +License. No additional rights or licenses will be implied from the distribution +or licensing of Covered Software under this License. Notwithstanding Section +2.1(b) above, no patent license is granted by a Contributor: + +for any code that a Contributor has removed from Covered Software; or + +for infringements caused by: (i) Your and any other third party’s +modifications of Covered Software, or (ii) the combination of its Contributions +with other software (except as part of its Contributor Version); or + +under Patent Claims infringed by Covered Software in the absence of its +Contributions. + +This License does not grant any rights in the trademarks, service marks, or +logos of any Contributor (except as may be necessary to comply with the notice +requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to distribute +the Covered Software under a subsequent version of this License (see Section +10.2) or under the terms of a Secondary License (if permitted under the terms of +Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its Contributions are +its original creation(s) or it has sufficient rights to grant the rights to its +Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under applicable +copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in +Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under the +terms of this License. You must inform recipients that the Source Code Form of +the Covered Software is governed by the terms of this License, and how they can +obtain a copy of this License. You may not attempt to alter or restrict the +recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +such Covered Software must also be made available in Source Code Form, as +described in Section 3.1, and You must inform recipients of the Executable Form +how they can obtain a copy of such Source Code Form by reasonable means in a +timely manner, at a charge no more than the cost of distribution to the +recipient; and + +You may distribute such Executable Form under the terms of this License, or +sublicense it under different terms, provided that the license for the +Executable Form does not attempt to limit or alter the recipients’ rights in +the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, provided +that You also comply with the requirements of this License for the Covered +Software. If the Larger Work is a combination of Covered Software with a work +governed by one or more Secondary Licenses, and the Covered Software is not +Incompatible With Secondary Licenses, this License permits You to additionally +distribute such Covered Software under the terms of such Secondary License(s), +so that the recipient of the Larger Work may, at their option, further +distribute the Covered Software under the terms of either this License or such +Secondary License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices (including +copyright notices, patent notices, disclaimers of warranty, or limitations of +liability) contained within the Source Code Form of the Covered Software, except +that You may alter any license notices to the extent required to remedy known +factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, indemnity +or liability obligations to one or more recipients of Covered Software. However, +You may do so only on Your own behalf, and not on behalf of any Contributor. You +must make it absolutely clear that any such warranty, support, indemnity, or +liability obligation is offered by You alone, and You hereby agree to indemnify +every Contributor for any liability incurred by such Contributor as a result of +warranty, support, indemnity or liability terms You offer. You may include +additional disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with +respect to some or all of the Covered Software due to statute, judicial order, +or regulation then You must: (a) comply with the terms of this License to the +maximum extent possible; and (b) describe the limitations and the code they +affect. Such description must be placed in a text file included with all +distributions of the Covered Software under this License. Except to the extent +prohibited by statute or regulation, such description must be sufficiently +detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You +fail to comply with any of its terms. However, if You become compliant, then the +rights granted under this License from a particular Contributor are reinstated +(a) provisionally, unless and until such Contributor explicitly and finally +terminates Your grants, and (b) on an ongoing basis, if such Contributor fails +to notify You of the non-compliance by some reasonable means prior to 60 days +after You have come back into compliance. Moreover, Your grants from a +particular Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the first +time You have received notice of non-compliance with this License from such +Contributor, and You become compliant prior to 30 days after Your receipt of the +notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, counter-claims, and +cross-claims) alleging that a Contributor Version directly or indirectly +infringes any patent, then the rights granted to You by any and all Contributors +for the Covered Software under Section 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user +license agreements (excluding distributors and resellers) which have been +validly granted by You or Your distributors under this License prior to +termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without +warranty of any kind, either expressed, implied, or statutory, including, +without limitation, warranties that the Covered Software is free of defects, +merchantable, fit for a particular purpose or non-infringing. The entire risk as +to the quality and performance of the Covered Software is with You. Should any +Covered Software prove defective in any respect, You (not any Contributor) +assume the cost of any necessary servicing, repair, or correction. This +disclaimer of warranty constitutes an essential part of this License. No use of +any Covered Software is authorized under this License except under this +disclaimer. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort (including +negligence), contract, or otherwise, shall any Contributor, or anyone who +distributes Covered Software as permitted above, be liable to You for any +direct, indirect, special, incidental, or consequential damages of any character +including, without limitation, damages for lost profits, loss of goodwill, work +stoppage, computer failure or malfunction, or any and all other commercial +damages or losses, even if such party shall have been informed of the +possibility of such damages. This limitation of liability shall not apply to +liability for death or personal injury resulting from such party’s negligence +to the extent applicable law prohibits such limitation. Some jurisdictions do +not allow the exclusion or limitation of incidental or consequential damages, so +this exclusion and limitation may not apply to You. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a +jurisdiction where the defendant maintains its principal place of business and +such litigation shall be governed by laws of that jurisdiction, without +reference to its conflict-of-law provisions. Nothing in this Section shall +prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter +hereof. If any provision of this License is held to be unenforceable, such +provision shall be reformed only to the extent necessary to make it enforceable. +Any law or regulation which provides that the language of a contract shall be +construed against the drafter shall not be used to construe this License against +a Contributor. + +10. Versions of the License + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section 10.3, +no one other than the license steward has the right to modify or publish new +versions of this License. Each version will be given a distinguishing version +number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version of the +License under which You originally received the Covered Software, or under the +terms of any subsequent version published by the license steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to create a +new license for such software, you may create and use a modified version of this +License if you rename the license and remove any references to the name of the +license steward (except to note that such modified license differs from this +License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + +If You choose to distribute Source Code Form that is Incompatible With Secondary +Licenses under the terms of this version of the License, the notice described in +Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not distributed with this file, You can obtain one +at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined +by the Mozilla Public License, v. 2.0. + +The binary distribution of this product bundles these dependencies under the +following license: +JDOM 1.1 +-------------------------------------------------------------------------------- +/*-- + + Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the disclaimer that follows + these conditions in the documentation and/or other materials + provided with the distribution. + + 3. The name "JDOM" must not be used to endorse or promote products + derived from this software without prior written permission. For + written permission, please contact . + + 4. Products derived from this software may not be called "JDOM", nor + may "JDOM" appear in their name, without prior written permission + from the JDOM Project Management . + + In addition, we request (but do not require) that you include in the + end-user documentation provided with the redistribution and/or in the + software itself an acknowledgement equivalent to the following: + "This product includes software developed by the + JDOM Project (http://www.jdom.org/)." + Alternatively, the acknowledgment may be graphical using the logos + available at http://www.jdom.org/images/logos. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + + This software consists of voluntary contributions made by many + individuals on behalf of the JDOM Project and was originally + created by Jason Hunter and + Brett McLaughlin . For more information + on the JDOM Project, please see . + + */ + +The binary distribution of this product bundles these dependencies under the +following license: +Hbase Server 1.2.4 +-------------------------------------------------------------------------------- +This project bundles a derivative image for our Orca Logo. This image is +available under the Creative Commons By Attribution 3.0 License. + + Creative Commons Legal Code + + Attribution 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + + License + + THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE + COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY + COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS + AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + + BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE + TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY + BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS + CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND + CONDITIONS. + + 1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined above) for the purposes of this + License. + c. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + d. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + e. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + f. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + g. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + h. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + i. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + + 2. Fair Dealing Rights. Nothing in this License is intended to reduce, + limit, or restrict any uses free from copyright or rights arising from + limitations or exceptions that are provided for in connection with the + copyright protection under copyright law or other applicable laws. + + 3. License Grant. Subject to the terms and conditions of this License, + Licensor hereby grants You a worldwide, royalty-free, non-exclusive, + perpetual (for the duration of the applicable copyright) license to + exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + + The above rights may be exercised in all media and formats whether now + known or hereafter devised. The above rights include the right to make + such modifications as are technically necessary to exercise the rights in + other media and formats. Subject to Section 8(f), all rights not expressly + granted by Licensor are hereby reserved. + + 4. Restrictions. The license granted in Section 3 above is expressly made + subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(b), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(b), as requested. + b. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Section 3(b), in the case of an Adaptation, + a credit identifying the use of the Work in the Adaptation (e.g., + "French translation of the Work by Original Author," or "Screenplay + based on original Work by Original Author"). The credit required by + this Section 4 (b) may be implemented in any reasonable manner; + provided, however, that in the case of a Adaptation or Collection, at + a minimum such credit will appear, if a credit for all contributing + authors of the Adaptation or Collection appears, then as part of these + credits and in a manner at least as prominent as the credits for the + other contributing authors. For the avoidance of doubt, You may only + use the credit required by this Section for the purpose of attribution + in the manner set out above and, by exercising Your rights under this + License, You may not implicitly or explicitly assert or imply any + connection with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of You or Your + use of the Work, without the separate, express prior written + permission of the Original Author, Licensor and/or Attribution + Parties. + c. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + + 5. Representations, Warranties and Disclaimer + + UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR + OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY + KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, + INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, + FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF + LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, + WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION + OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + + 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE + LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR + ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES + ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS + BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + + 8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + + Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of this License. + + Creative Commons may be contacted at https://creativecommons.org/. +-------------------------------------------------------------------------------- + +For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/AbstractFuture.java and +hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/TimeoutFuture.java + +Copyright (C) 2007 The Guava Authors + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-re2j b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-re2j new file mode 100644 index 0000000000000..b620ae68fe332 --- /dev/null +++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-re2j @@ -0,0 +1,32 @@ +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-stax2api b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-stax2api new file mode 100644 index 0000000000000..0ed6361699658 --- /dev/null +++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/licenses/LICENSE-stax2api @@ -0,0 +1,22 @@ +Copyright woodstox stax2api contributors. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableFsDataOutputStream.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableFsDataOutputStream.java index c688b3217c4ec..be0d134b54e1c 100644 --- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableFsDataOutputStream.java +++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableFsDataOutputStream.java @@ -86,14 +86,8 @@ class HadoopRecoverableFsDataOutputStream extends RecoverableFsDataOutputStream this.targetFile = checkNotNull(recoverable.targetFile()); this.tempFile = checkNotNull(recoverable.tempFile()); - // truncate back and append - try { - truncate(fs, tempFile, recoverable.offset()); - } catch (Exception e) { - throw new IOException("Missing data in tmp file: " + tempFile, e); - } + safelyTruncateFile(fs, tempFile, recoverable); - waitUntilLeaseIsRevoked(tempFile); out = fs.append(tempFile); // sanity check @@ -155,6 +149,30 @@ public void close() throws IOException { // Hadoop 2.7, which have no truncation calls for HDFS. // ------------------------------------------------------------------------ + private static void safelyTruncateFile( + final FileSystem fileSystem, + final Path path, + final HadoopFsRecoverable recoverable) throws IOException { + + ensureTruncateInitialized(); + + waitUntilLeaseIsRevoked(fileSystem, path); + + // truncate back and append + boolean truncated; + try { + truncated = truncate(fileSystem, path, recoverable.offset()); + } catch (Exception e) { + throw new IOException("Problem while truncating file: " + path, e); + } + + if (!truncated) { + // Truncate did not complete immediately, we must wait for + // the operation to complete and release the lease. + waitUntilLeaseIsRevoked(fileSystem, path); + } + } + private static void ensureTruncateInitialized() throws FlinkRuntimeException { if (truncateHandle == null) { Method truncateMethod; @@ -173,10 +191,10 @@ private static void ensureTruncateInitialized() throws FlinkRuntimeException { } } - static void truncate(FileSystem hadoopFs, Path file, long length) throws IOException { + private static boolean truncate(final FileSystem hadoopFs, final Path file, final long length) throws IOException { if (truncateHandle != null) { try { - truncateHandle.invoke(hadoopFs, file, length); + return (Boolean) truncateHandle.invoke(hadoopFs, file, length); } catch (InvocationTargetException e) { ExceptionUtils.rethrowIOException(e.getTargetException()); @@ -190,6 +208,7 @@ static void truncate(FileSystem hadoopFs, Path file, long length) throws IOExcep else { throw new IllegalStateException("Truncation handle has not been initialized"); } + return false; } // ------------------------------------------------------------------------ @@ -260,12 +279,7 @@ public void commitAfterRecovery() throws IOException { if (srcStatus.getLen() > expectedLength) { // can happen if we go from persist to recovering for commit directly // truncate the trailing junk away - try { - truncate(fs, src, expectedLength); - } catch (Exception e) { - // this can happen if the file is smaller than expected - throw new IOException("Problem while truncating file: " + src, e); - } + safelyTruncateFile(fs, src, recoverable); } // rename to final location (if it exists, overwrite it) @@ -304,7 +318,7 @@ public CommitRecoverable getRecoverable() { * * @param path The path to the file we want to resume writing to. */ - private boolean waitUntilLeaseIsRevoked(final Path path) throws IOException { + private static boolean waitUntilLeaseIsRevoked(final FileSystem fs, final Path path) throws IOException { Preconditions.checkState(fs instanceof DistributedFileSystem); final DistributedFileSystem dfs = (DistributedFileSystem) fs; diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java index 305f8ee5612ef..03d741b4b82b5 100644 --- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java +++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java @@ -77,6 +77,16 @@ public RecoverableFsDataOutputStream recover(ResumeRecoverable recoverable) thro } } + @Override + public boolean requiresCleanupOfRecoverableState() { + return false; + } + + @Override + public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public Committer recoverForCommit(CommitRecoverable recoverable) throws IOException { if (recoverable instanceof HadoopFsRecoverable) { diff --git a/flink-filesystems/flink-s3-fs-base/pom.xml b/flink-filesystems/flink-s3-fs-base/pom.xml index 6cc877dd57305..30fceadd57832 100644 --- a/flink-filesystems/flink-s3-fs-base/pom.xml +++ b/flink-filesystems/flink-s3-fs-base/pom.xml @@ -177,6 +177,7 @@ under the License. mime.types mozilla/** META-INF/maven/** + META-INF/LICENSE.txt diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java index 318fd39829c79..ff575be6f55c8 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java @@ -21,11 +21,12 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.IllegalConfigurationException; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystemFactory; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; +import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,10 +139,12 @@ public FileSystem create(URI fsUri) throws IOException { } } - final String localTmpDirectory = flinkConfig.getString(CoreOptions.TMP_DIRS); + final String[] localTmpDirectories = ConfigurationUtils.parseTempDirectories(flinkConfig); + Preconditions.checkArgument(localTmpDirectories.length > 0); + final String localTmpDirectory = localTmpDirectories[0]; final long s3minPartSize = flinkConfig.getLong(PART_UPLOAD_MIN_SIZE); final int maxConcurrentUploads = flinkConfig.getInteger(MAX_CONCURRENT_UPLOADS); - final S3MultiPartUploader s3AccessHelper = getS3AccessHelper(fs); + final S3AccessHelper s3AccessHelper = getS3AccessHelper(fs); return new FlinkS3FileSystem( fs, @@ -166,6 +169,6 @@ protected abstract URI getInitURI( URI fsUri, org.apache.hadoop.conf.Configuration hadoopConfig); @Nullable - protected abstract S3MultiPartUploader getS3AccessHelper(org.apache.hadoop.fs.FileSystem fs); + protected abstract S3AccessHelper getS3AccessHelper(org.apache.hadoop.fs.FileSystem fs); } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java index 553edde75b06c..5248e061a12ec 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java @@ -23,7 +23,7 @@ import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.fs.s3.common.utils.RefCountedFile; import org.apache.flink.fs.s3.common.utils.RefCountedTmpFileCreator; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.flink.fs.s3.common.writer.S3RecoverableWriter; import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; import org.apache.flink.util.Preconditions; @@ -60,7 +60,7 @@ public class FlinkS3FileSystem extends HadoopFileSystem implements EntropyInject private final FunctionWithException tmpFileCreator; @Nullable - private final S3MultiPartUploader s3UploadHelper; + private final S3AccessHelper s3AccessHelper; private final Executor uploadThreadPool; @@ -83,7 +83,7 @@ public FlinkS3FileSystem( String localTmpDirectory, @Nullable String entropyInjectionKey, int entropyLength, - @Nullable S3MultiPartUploader s3UploadHelper, + @Nullable S3AccessHelper s3UploadHelper, long s3uploadPartSize, int maxConcurrentUploadsPerStream) { @@ -99,7 +99,7 @@ public FlinkS3FileSystem( // recoverable writer parameter configuration initialization this.localTmpDir = Preconditions.checkNotNull(localTmpDirectory); this.tmpFileCreator = RefCountedTmpFileCreator.inDirectories(new File(localTmpDirectory)); - this.s3UploadHelper = s3UploadHelper; + this.s3AccessHelper = s3UploadHelper; this.uploadThreadPool = Executors.newCachedThreadPool(); Preconditions.checkArgument(s3uploadPartSize >= S3_MULTIPART_MIN_PART_SIZE); @@ -131,7 +131,7 @@ public String getLocalTmpDir() { @Override public RecoverableWriter createRecoverableWriter() throws IOException { - if (s3UploadHelper == null) { + if (s3AccessHelper == null) { // this is the case for Presto throw new UnsupportedOperationException("This s3 file system implementation does not support recoverable writers."); } @@ -139,7 +139,7 @@ public RecoverableWriter createRecoverableWriter() throws IOException { return S3RecoverableWriter.writer( getHadoopFileSystem(), tmpFileCreator, - s3UploadHelper, + s3AccessHelper, uploadThreadPool, s3uploadPartSize, maxConcurrentUploadsPerStream); diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/BackPressuringExecutor.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/BackPressuringExecutor.java index d0dd7c8f73440..e78e83de4252e 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/BackPressuringExecutor.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/BackPressuringExecutor.java @@ -23,7 +23,6 @@ import org.apache.flink.util.FlinkRuntimeException; import java.util.concurrent.Executor; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; @@ -66,11 +65,9 @@ public void execute(Runnable command) { final SemaphoreReleasingRunnable runnable = new SemaphoreReleasingRunnable(command, permits); try { delegate.execute(runnable); - } catch (RejectedExecutionException e) { + } catch (Throwable e) { runnable.release(); ExceptionUtils.rethrow(e, e.getMessage()); - } catch (Throwable t) { - ExceptionUtils.rethrow(t, t.getMessage()); } } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java index 8f3aff899f1a1..29f2590803cc8 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java @@ -24,9 +24,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -64,8 +61,8 @@ public RefCountedBufferingFileStream( } @Override - public InputStream getInputStream() throws IOException { - return Files.newInputStream(currentTmpFile.getFile().toPath(), StandardOpenOption.READ); + public File getInputFile() { + return currentTmpFile.getFile(); } @Override diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java index d4b962e146396..d51e37e8bfbf3 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java @@ -21,8 +21,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.FSDataOutputStream; +import java.io.File; import java.io.IOException; -import java.io.InputStream; /** * A {@link FSDataOutputStream} with the {@link RefCounted} functionality. @@ -31,11 +31,11 @@ public abstract class RefCountedFSOutputStream extends FSDataOutputStream implements RefCounted { /** - * Gets an {@link InputStream} that allows to read the contents of the file. + * Gets the underlying {@link File} that allows to read the contents of the file. * - * @return An input stream to the contents of the file. + * @return A handle to the File object. */ - public abstract InputStream getInputStream() throws IOException; + public abstract File getInputFile(); /** * Checks if the file is closed for writes. diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java index 80042ce22f78b..0d0998a062002 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; @@ -58,7 +57,7 @@ @NotThreadSafe final class RecoverableMultiPartUploadImpl implements RecoverableMultiPartUpload { - private final S3MultiPartUploader s3MPUploader; + private final S3AccessHelper s3AccessHelper; private final Executor uploadThreadPool; @@ -71,7 +70,7 @@ final class RecoverableMultiPartUploadImpl implements RecoverableMultiPartUpload // ------------------------------------------------------------------------ private RecoverableMultiPartUploadImpl( - S3MultiPartUploader s3uploader, + S3AccessHelper s3AccessHelper, Executor uploadThreadPool, String uploadId, String objectName, @@ -81,10 +80,10 @@ private RecoverableMultiPartUploadImpl( ) { checkArgument(numBytes >= 0L); - this.s3MPUploader = checkNotNull(s3uploader); + this.s3AccessHelper = checkNotNull(s3AccessHelper); this.uploadThreadPool = checkNotNull(uploadThreadPool); this.currentUploadInfo = new MultiPartUploadInfo(objectName, uploadId, partsSoFar, numBytes, incompletePart); - this.namePrefixForTempObjects = incompleteObjectNamePrefix(objectName); + this.namePrefixForTempObjects = createIncompletePartObjectNamePrefix(objectName); this.uploadsInProgress = new ArrayDeque<>(); } @@ -111,7 +110,7 @@ public void uploadPart(RefCountedFSOutputStream file) throws IOException { currentUploadInfo.registerNewPart(partLength); file.retain(); // keep the file while the async upload still runs - uploadThreadPool.execute(new UploadTask(s3MPUploader, currentUploadInfo, file, future)); + uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future)); } @Override @@ -124,7 +123,7 @@ public S3Committer snapshotAndGetCommitter() throws IOException { final S3Recoverable snapshot = snapshotAndGetRecoverable(null); return new S3Committer( - s3MPUploader, + s3AccessHelper, snapshot.getObjectName(), snapshot.uploadId(), snapshot.parts(), @@ -171,15 +170,10 @@ private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) th } // first, upload the trailing data file. during that time, other in-progress uploads may complete. - final String incompletePartObjectName = createTmpObjectName(); + final String incompletePartObjectName = createIncompletePartObjectName(); file.retain(); - try (InputStream inputStream = file.getInputStream()) { - - // TODO: staged incomplete parts are not cleaned up as - // they do not fall under the user's global TTL on S3. - // Figure out a way to clean them. - - s3MPUploader.uploadIncompletePart(incompletePartObjectName, inputStream, file.getPos()); + try { + s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile()); } finally { file.release(); @@ -192,7 +186,7 @@ private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) th // ------------------------------------------------------------------------ @VisibleForTesting - static String incompleteObjectNamePrefix(String objectName) { + static String createIncompletePartObjectNamePrefix(String objectName) { checkNotNull(objectName); final int lastSlash = objectName.lastIndexOf('/'); @@ -209,6 +203,10 @@ static String incompleteObjectNamePrefix(String objectName) { return parent + (child.isEmpty() ? "" : '_') + child + "_tmp_"; } + private String createIncompletePartObjectName() { + return namePrefixForTempObjects + UUID.randomUUID().toString(); + } + private void awaitPendingPartsUpload() throws IOException { checkState(currentUploadInfo.getRemainingParts() == uploadsInProgress.size()); @@ -235,23 +233,19 @@ private PartETag awaitPendingPartUploadToComplete(CompletableFuture up return completedUploadEtag; } - private String createTmpObjectName() { - return namePrefixForTempObjects + UUID.randomUUID().toString(); - } - // ------------------------------------------------------------------------ // factory methods // ------------------------------------------------------------------------ public static RecoverableMultiPartUploadImpl newUpload( - final S3MultiPartUploader s3uploader, + final S3AccessHelper s3AccessHelper, final Executor uploadThreadPool, final String objectName) throws IOException { - final String multiPartUploadId = s3uploader.startMultiPartUpload(objectName); + final String multiPartUploadId = s3AccessHelper.startMultiPartUpload(objectName); return new RecoverableMultiPartUploadImpl( - s3uploader, + s3AccessHelper, uploadThreadPool, multiPartUploadId, objectName, @@ -261,7 +255,7 @@ public static RecoverableMultiPartUploadImpl newUpload( } public static RecoverableMultiPartUploadImpl recoverUpload( - final S3MultiPartUploader s3uploader, + final S3AccessHelper s3AccessHelper, final Executor uploadThreadPool, final String multipartUploadId, final String objectName, @@ -270,7 +264,7 @@ public static RecoverableMultiPartUploadImpl recoverUpload( final Optional incompletePart) { return new RecoverableMultiPartUploadImpl( - s3uploader, + s3AccessHelper, uploadThreadPool, multipartUploadId, objectName, @@ -286,7 +280,7 @@ public static RecoverableMultiPartUploadImpl recoverUpload( private static class UploadTask implements Runnable { - private final S3MultiPartUploader s3uploader; + private final S3AccessHelper s3AccessHelper; private final String objectName; @@ -299,7 +293,7 @@ private static class UploadTask implements Runnable { private final CompletableFuture future; UploadTask( - final S3MultiPartUploader s3uploader, + final S3AccessHelper s3AccessHelper, final MultiPartUploadInfo currentUpload, final RefCountedFSOutputStream file, final CompletableFuture future) { @@ -313,15 +307,15 @@ private static class UploadTask implements Runnable { // these are limits put by Amazon checkArgument(partNumber >= 1 && partNumber <= 10_000); - this.s3uploader = checkNotNull(s3uploader); + this.s3AccessHelper = checkNotNull(s3AccessHelper); this.file = checkNotNull(file); this.future = checkNotNull(future); } @Override public void run() { - try (final InputStream inputStream = file.getInputStream()) { - final UploadPartResult result = s3uploader.uploadPart(objectName, uploadId, partNumber, inputStream, file.getPos()); + try { + final UploadPartResult result = s3AccessHelper.uploadPart(objectName, uploadId, partNumber, file.getInputFile(), file.getPos()); future.complete(new PartETag(result.getPartNumber(), result.getETag())); file.release(); } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3MultiPartUploader.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java similarity index 73% rename from flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3MultiPartUploader.java rename to flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java index da227a47488e6..593d9d36de2c1 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3MultiPartUploader.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java @@ -26,8 +26,8 @@ import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.UploadPartResult; +import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -41,7 +41,7 @@ * the upload with all its parts will be either committed or discarded. */ @Internal -public interface S3MultiPartUploader { +public interface S3AccessHelper { /** * Initializes a Multi-Part Upload. @@ -58,26 +58,24 @@ public interface S3MultiPartUploader { * @param key the key this MPU is associated with. * @param uploadId the id of the MPU. * @param partNumber the number of the part being uploaded (has to be in [1 ... 10000]). - * @param file the (local) file holding the part to be uploaded. + * @param inputFile the (local) file holding the part to be uploaded. * @param length the length of the part. * @return The {@link UploadPartResult result} of the attempt to upload the part. * @throws IOException */ - UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream file, long length) throws IOException; + UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException; /** - * Uploads a part and associates it with the MPU with the provided {@code uploadId}. - * - *

    Contrary to the {@link #uploadIncompletePart(String, InputStream, long)}, this part can - * be smaller than the minimum part size imposed by S3. + * Uploads an object to S3. Contrary to the {@link #uploadPart(String, String, int, File, long)} method, + * this object is not going to be associated to any MPU and, as such, it is not subject to the garbage collection + * policies specified for your S3 bucket. * * @param key the key used to identify this part. - * @param file the (local) file holding the data to be uploaded. - * @param length the size of the data to be uploaded. + * @param inputFile the (local) file holding the data to be uploaded. * @return The {@link PutObjectResult result} of the attempt to stage the incomplete part. * @throws IOException */ - PutObjectResult uploadIncompletePart(String key, InputStream file, long length) throws IOException; + PutObjectResult putObject(String key, File inputFile) throws IOException; /** * Finalizes a Multi-Part Upload. @@ -92,6 +90,27 @@ public interface S3MultiPartUploader { */ CompleteMultipartUploadResult commitMultiPartUpload(String key, String uploadId, List partETags, long length, AtomicInteger errorCount) throws IOException; + /** + * Deletes the object associated with the provided key. + * + * @param key The key to be deleted. + * @return {@code true} if the resources were successfully freed, {@code false} otherwise + * (e.g. the file to be deleted was not there). + * @throws IOException + */ + boolean deleteObject(String key) throws IOException; + + /** + * Gets the object associated with the provided {@code key} from S3 and + * puts it in the provided {@code targetLocation}. + * + * @param key the key of the object to fetch. + * @param targetLocation the file to read the object to. + * @return The number of bytes read. + * @throws IOException + */ + long getObject(String key, File targetLocation) throws IOException; + /** * Fetches the metadata associated with a given key on S3. * diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3Committer.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3Committer.java index 1fc8bf1eaa8dd..5fbc5bb5d42de 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3Committer.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3Committer.java @@ -40,7 +40,7 @@ public final class S3Committer implements RecoverableFsDataOutputStream.Committe private static final Logger LOG = LoggerFactory.getLogger(S3Committer.class); - private final S3MultiPartUploader s3uploader; + private final S3AccessHelper s3AccessHelper; private final String uploadId; @@ -50,8 +50,8 @@ public final class S3Committer implements RecoverableFsDataOutputStream.Committe private final long totalLength; - S3Committer(S3MultiPartUploader s3uploader, String objectName, String uploadId, List parts, long totalLength) { - this.s3uploader = checkNotNull(s3uploader); + S3Committer(S3AccessHelper s3AccessHelper, String objectName, String uploadId, List parts, long totalLength) { + this.s3AccessHelper = checkNotNull(s3AccessHelper); this.objectName = checkNotNull(objectName); this.uploadId = checkNotNull(uploadId); this.parts = checkNotNull(parts); @@ -64,7 +64,7 @@ public void commit() throws IOException { LOG.info("Committing {} with MPU ID {}", objectName, uploadId); final AtomicInteger errorCount = new AtomicInteger(); - s3uploader.commitMultiPartUpload(objectName, uploadId, parts, totalLength, errorCount); + s3AccessHelper.commitMultiPartUpload(objectName, uploadId, parts, totalLength, errorCount); if (errorCount.get() == 0) { LOG.debug("Successfully committed {} with MPU ID {}", objectName, uploadId); @@ -82,14 +82,14 @@ public void commitAfterRecovery() throws IOException { LOG.info("Trying to commit after recovery {} with MPU ID {}", objectName, uploadId); try { - s3uploader.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger()); + s3AccessHelper.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger()); } catch (IOException e) { LOG.info("Failed to commit after recovery {} with MPU ID {}. " + "Checking if file was committed before...", objectName, uploadId); LOG.trace("Exception when committing:", e); try { - ObjectMetadata metadata = s3uploader.getObjectMetadata(objectName); + ObjectMetadata metadata = s3AccessHelper.getObjectMetadata(objectName); if (totalLength != metadata.getContentLength()) { String message = String.format("Inconsistent result for object %s: conflicting lengths. " + "Recovered committer for upload %s indicates %s bytes, present object is %s bytes", diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java index b201981f31a04..3727e25790437 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java @@ -19,10 +19,8 @@ package org.apache.flink.fs.s3.common.writer; import org.apache.flink.annotation.Internal; -import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.Path; import org.apache.flink.fs.s3.common.utils.BackPressuringExecutor; -import org.apache.flink.fs.s3.common.utils.OffsetAwareOutputStream; import org.apache.flink.fs.s3.common.utils.RefCountedFile; import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; import org.apache.flink.util.Preconditions; @@ -43,7 +41,7 @@ final class S3RecoverableMultipartUploadFactory { private final org.apache.hadoop.fs.FileSystem fs; - private final S3MultiPartUploader twoPhaseUploader; + private final S3AccessHelper s3AccessHelper; private final FunctionWithException tmpFileSupplier; @@ -53,7 +51,7 @@ final class S3RecoverableMultipartUploadFactory { S3RecoverableMultipartUploadFactory( final FileSystem fs, - final S3MultiPartUploader twoPhaseUploader, + final S3AccessHelper s3AccessHelper, final int maxConcurrentUploadsPerStream, final Executor executor, final FunctionWithException tmpFileSupplier) { @@ -61,23 +59,23 @@ final class S3RecoverableMultipartUploadFactory { this.fs = Preconditions.checkNotNull(fs); this.maxConcurrentUploadsPerStream = maxConcurrentUploadsPerStream; this.executor = executor; - this.twoPhaseUploader = twoPhaseUploader; + this.s3AccessHelper = s3AccessHelper; this.tmpFileSupplier = tmpFileSupplier; } RecoverableMultiPartUpload getNewRecoverableUpload(Path path) throws IOException { return RecoverableMultiPartUploadImpl.newUpload( - twoPhaseUploader, + s3AccessHelper, limitedExecutor(), pathToObjectName(path)); } RecoverableMultiPartUpload recoverRecoverableUpload(S3Recoverable recoverable) throws IOException { - final Optional incompletePart = downloadLastDataChunk(recoverable); + final Optional incompletePart = recoverInProgressPart(recoverable); return RecoverableMultiPartUploadImpl.recoverUpload( - twoPhaseUploader, + s3AccessHelper, limitedExecutor(), recoverable.uploadId(), recoverable.getObjectName(), @@ -86,41 +84,17 @@ RecoverableMultiPartUpload recoverRecoverableUpload(S3Recoverable recoverable) t incompletePart); } - @VisibleForTesting - Optional downloadLastDataChunk(S3Recoverable recoverable) throws IOException { + private Optional recoverInProgressPart(S3Recoverable recoverable) throws IOException { - final String objectName = recoverable.incompleteObjectName(); - if (objectName == null) { + final String objectKey = recoverable.incompleteObjectName(); + if (objectKey == null) { return Optional.empty(); } // download the file (simple way) - final RefCountedFile fileAndStream = tmpFileSupplier.apply(null); - final File file = fileAndStream.getFile(); - - long numBytes = 0L; - - try ( - final OffsetAwareOutputStream outStream = fileAndStream.getStream(); - final org.apache.hadoop.fs.FSDataInputStream inStream = - fs.open(new org.apache.hadoop.fs.Path('/' + objectName)) - ) { - final byte[] buffer = new byte[32 * 1024]; - - int numRead; - while ((numRead = inStream.read(buffer)) > 0) { - outStream.write(buffer, 0, numRead); - numBytes += numRead; - } - } - - // some sanity checks - if (numBytes != file.length() || numBytes != fileAndStream.getStream().getLength()) { - throw new IOException(String.format("Error recovering writer: " + - "Downloading the last data chunk file gives incorrect length. " + - "File=%d bytes, Stream=%d bytes", - file.length(), numBytes)); - } + final RefCountedFile refCountedFile = tmpFileSupplier.apply(null); + final File file = refCountedFile.getFile(); + final long numBytes = s3AccessHelper.getObject(objectKey, file); if (numBytes != recoverable.incompleteObjectLength()) { throw new IOException(String.format("Error recovering writer: " + @@ -132,8 +106,7 @@ Optional downloadLastDataChunk(S3Recoverable recoverable) throws IOExcepti return Optional.of(file); } - @VisibleForTesting - String pathToObjectName(final Path path) { + private String pathToObjectName(final Path path) { org.apache.hadoop.fs.Path hadoopPath = HadoopFileSystem.toHadoopPath(path); if (!hadoopPath.isAbsolute()) { hadoopPath = new org.apache.hadoop.fs.Path(fs.getWorkingDirectory(), hadoopPath); diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java index 2a84308d5b01d..ddb4443c58564 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java @@ -26,7 +26,6 @@ import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.fs.s3.common.utils.RefCountedFile; -import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.FunctionWithException; import org.apache.hadoop.fs.FileSystem; @@ -37,6 +36,7 @@ import static org.apache.flink.fs.s3.common.FlinkS3FileSystem.S3_MULTIPART_MIN_PART_SIZE; import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; /** * An implementation of the {@link RecoverableWriter} against S3. @@ -54,16 +54,20 @@ public class S3RecoverableWriter implements RecoverableWriter { private final long userDefinedMinPartSize; + private final S3AccessHelper s3AccessHelper; + private final S3RecoverableMultipartUploadFactory uploadFactory; @VisibleForTesting S3RecoverableWriter( + final S3AccessHelper s3AccessHelper, final S3RecoverableMultipartUploadFactory uploadFactory, final FunctionWithException tempFileCreator, final long userDefinedMinPartSize) { - this.uploadFactory = Preconditions.checkNotNull(uploadFactory); - this.tempFileCreator = Preconditions.checkNotNull(tempFileCreator); + this.s3AccessHelper = checkNotNull(s3AccessHelper); + this.uploadFactory = checkNotNull(uploadFactory); + this.tempFileCreator = checkNotNull(tempFileCreator); this.userDefinedMinPartSize = userDefinedMinPartSize; } @@ -78,14 +82,14 @@ public RecoverableFsDataOutputStream open(Path path) throws IOException { } @Override - public Committer recoverForCommit(RecoverableWriter.CommitRecoverable recoverable) throws IOException { + public Committer recoverForCommit(CommitRecoverable recoverable) throws IOException { final S3Recoverable s3recoverable = castToS3Recoverable(recoverable); final S3RecoverableFsDataOutputStream recovered = recover(s3recoverable); return recovered.closeForCommit(); } @Override - public S3RecoverableFsDataOutputStream recover(RecoverableWriter.ResumeRecoverable recoverable) throws IOException { + public S3RecoverableFsDataOutputStream recover(ResumeRecoverable recoverable) throws IOException { final S3Recoverable s3recoverable = castToS3Recoverable(recoverable); final RecoverableMultiPartUpload upload = uploadFactory.recoverRecoverableUpload(s3recoverable); @@ -97,15 +101,27 @@ public S3RecoverableFsDataOutputStream recover(RecoverableWriter.ResumeRecoverab s3recoverable.numBytesInParts()); } + @Override + public boolean requiresCleanupOfRecoverableState() { + return true; + } + + @Override + public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { + final S3Recoverable s3recoverable = castToS3Recoverable(resumable); + final String smallPartObjectToDelete = s3recoverable.incompleteObjectName(); + return smallPartObjectToDelete != null && s3AccessHelper.deleteObject(smallPartObjectToDelete); + } + @Override @SuppressWarnings({"rawtypes", "unchecked"}) - public SimpleVersionedSerializer getCommitRecoverableSerializer() { + public SimpleVersionedSerializer getCommitRecoverableSerializer() { return (SimpleVersionedSerializer) S3RecoverableSerializer.INSTANCE; } @Override @SuppressWarnings({"rawtypes", "unchecked"}) - public SimpleVersionedSerializer getResumeRecoverableSerializer() { + public SimpleVersionedSerializer getResumeRecoverableSerializer() { return (SimpleVersionedSerializer) S3RecoverableSerializer.INSTANCE; } @@ -116,7 +132,7 @@ public boolean supportsResume() { // --------------------------- Utils --------------------------- - private static S3Recoverable castToS3Recoverable(RecoverableWriter.CommitRecoverable recoverable) { + private static S3Recoverable castToS3Recoverable(CommitRecoverable recoverable) { if (recoverable instanceof S3Recoverable) { return (S3Recoverable) recoverable; } @@ -129,7 +145,7 @@ private static S3Recoverable castToS3Recoverable(RecoverableWriter.CommitRecover public static S3RecoverableWriter writer( final FileSystem fs, final FunctionWithException tempFileCreator, - final S3MultiPartUploader twoPhaseUploader, + final S3AccessHelper s3AccessHelper, final Executor uploadThreadPool, final long userDefinedMinPartSize, final int maxConcurrentUploadsPerStream) { @@ -139,11 +155,11 @@ public static S3RecoverableWriter writer( final S3RecoverableMultipartUploadFactory uploadFactory = new S3RecoverableMultipartUploadFactory( fs, - twoPhaseUploader, + s3AccessHelper, maxConcurrentUploadsPerStream, uploadThreadPool, tempFileCreator); - return new S3RecoverableWriter(uploadFactory, tempFileCreator, userDefinedMinPartSize); + return new S3RecoverableWriter(s3AccessHelper, uploadFactory, tempFileCreator, userDefinedMinPartSize); } } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-base/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..eed06551bedec --- /dev/null +++ b/flink-filesystems/flink-s3-fs-base/src/main/resources/META-INF/NOTICE @@ -0,0 +1,24 @@ +flink-s3-fs-base +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-aws:3.1.0 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpclient:4.5.3 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- com.amazonaws:aws-java-sdk-core:1.11.271 +- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 +- com.amazonaws:aws-java-sdk-kms:1.11.271 +- com.amazonaws:aws-java-sdk-s3:1.11.271 +- com.amazonaws:jmespath-java:1.11.271 +- software.amazon.ion:ion-java:1.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.6.0 +- com.fasterxml.jackson.core:jackson-core:2.6.7 +- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 +- joda-time:joda-time:2.5 diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java index d3d25c347e8ca..943de1d889727 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java @@ -19,7 +19,7 @@ package org.apache.flink.fs.s3.common; import org.apache.flink.configuration.Configuration; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.flink.util.TestLogger; import org.apache.hadoop.fs.FileSystem; @@ -52,6 +52,24 @@ public void testEntropyInjectionConfig() throws Exception { assertEquals(7, fs.generateEntropy().length()); } + /** + * Test validates that the produced by AbstractS3FileSystemFactory object will contains + * only first path from multiple paths in config. + */ + @Test + public void testMultipleTempDirsConfig() throws Exception { + final Configuration conf = new Configuration(); + String dir1 = "/tmp/dir1"; + String dir2 = "/tmp/dir2"; + conf.setString("io.tmp.dirs", dir1 + "," + dir2); + + TestFsFactory factory = new TestFsFactory(); + factory.configure(conf); + + FlinkS3FileSystem fs = (FlinkS3FileSystem) factory.create(new URI("s3://test")); + assertEquals(fs.getLocalTmpDir(), dir1); + } + // ------------------------------------------------------------------------ private static final class TestFsFactory extends AbstractS3FileSystemFactory { @@ -78,7 +96,7 @@ protected URI getInitURI(URI fsUri, org.apache.hadoop.conf.Configuration hadoopC @Nullable @Override - protected S3MultiPartUploader getS3AccessHelper(FileSystem fs) { + protected S3AccessHelper getS3AccessHelper(FileSystem fs) { return null; } diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java index 5b7d1ccfb9f2d..50ea9bd64a6e1 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java @@ -24,6 +24,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -121,7 +122,7 @@ public void testFlush() throws IOException { Assert.assertEquals(contentToWrite.length, stream.getPos()); final byte[] contentRead = new byte[contentToWrite.length]; - stream.getInputStream().read(contentRead, 0, contentRead.length); + new FileInputStream(stream.getInputFile()).read(contentRead, 0, contentRead.length); Assert.assertTrue(Arrays.equals(contentToWrite, contentRead)); stream.release(); diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/IncompletePartPrefixTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/IncompletePartPrefixTest.java index a3164f18a16d2..9cee040c067fd 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/IncompletePartPrefixTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/IncompletePartPrefixTest.java @@ -22,36 +22,36 @@ import org.junit.Test; /** - * Tests for the {@link RecoverableMultiPartUploadImpl#incompleteObjectNamePrefix(String)}. + * Tests for the {@link RecoverableMultiPartUploadImpl#createIncompletePartObjectNamePrefix(String)}. */ public class IncompletePartPrefixTest { @Test(expected = NullPointerException.class) public void nullObjectNameShouldThroughException() { - RecoverableMultiPartUploadImpl.incompleteObjectNamePrefix(null); + RecoverableMultiPartUploadImpl.createIncompletePartObjectNamePrefix(null); } @Test public void emptyInitialNameShouldSucceed() { - String objectNamePrefix = RecoverableMultiPartUploadImpl.incompleteObjectNamePrefix(""); + String objectNamePrefix = RecoverableMultiPartUploadImpl.createIncompletePartObjectNamePrefix(""); Assert.assertEquals("_tmp_", objectNamePrefix); } @Test public void nameWithoutSlashShouldSucceed() { - String objectNamePrefix = RecoverableMultiPartUploadImpl.incompleteObjectNamePrefix("no_slash_path"); + String objectNamePrefix = RecoverableMultiPartUploadImpl.createIncompletePartObjectNamePrefix("no_slash_path"); Assert.assertEquals("_no_slash_path_tmp_", objectNamePrefix); } @Test public void nameWithOnlySlashShouldSucceed() { - String objectNamePrefix = RecoverableMultiPartUploadImpl.incompleteObjectNamePrefix("/"); + String objectNamePrefix = RecoverableMultiPartUploadImpl.createIncompletePartObjectNamePrefix("/"); Assert.assertEquals("/_tmp_", objectNamePrefix); } @Test public void normalPathShouldSucceed() { - String objectNamePrefix = RecoverableMultiPartUploadImpl.incompleteObjectNamePrefix("/root/home/test-file"); + String objectNamePrefix = RecoverableMultiPartUploadImpl.createIncompletePartObjectNamePrefix("/root/home/test-file"); Assert.assertEquals("/root/home/_test-file_tmp_", objectNamePrefix); } } diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java index 72554e197739f..0194065633f30 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java @@ -35,8 +35,8 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -339,19 +339,19 @@ public void execute(Runnable command) { } /** - * A {@link S3MultiPartUploader} that simulates uploading part files to S3 by + * A {@link S3AccessHelper} that simulates uploading part files to S3 by * simply putting complete and incomplete part files in lists for further validation. */ - private static class StubMultiPartUploader implements S3MultiPartUploader { + private static class StubMultiPartUploader implements S3AccessHelper { private final List completePartsUploaded = new ArrayList<>(); private final List incompletePartsUploaded = new ArrayList<>(); - public List getCompletePartsUploaded() { + List getCompletePartsUploaded() { return completePartsUploaded; } - public List getIncompletePartsUploaded() { + List getIncompletePartsUploaded() { return incompletePartsUploaded; } @@ -361,17 +361,27 @@ public String startMultiPartUpload(String key) throws IOException { } @Override - public UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream file, long length) throws IOException { - final byte[] content = getFileContentBytes(file, MathUtils.checkedDownCast(length)); + public UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException { + final byte[] content = getFileContentBytes(inputFile, MathUtils.checkedDownCast(length)); return storeAndGetUploadPartResult(key, partNumber, content); } @Override - public PutObjectResult uploadIncompletePart(String key, InputStream file, long length) throws IOException { - final byte[] content = getFileContentBytes(file, MathUtils.checkedDownCast(length)); + public PutObjectResult putObject(String key, File inputFile) throws IOException { + final byte[] content = getFileContentBytes(inputFile, MathUtils.checkedDownCast(inputFile.length())); return storeAndGetPutObjectResult(key, content); } + @Override + public boolean deleteObject(String key) throws IOException { + return false; + } + + @Override + public long getObject(String key, File targetLocation) throws IOException { + return 0; + } + @Override public CompleteMultipartUploadResult commitMultiPartUpload( String key, @@ -387,9 +397,9 @@ public ObjectMetadata getObjectMetadata(String key) throws IOException { return null; } - private byte[] getFileContentBytes(InputStream file, int length) throws IOException { + private byte[] getFileContentBytes(File file, int length) throws IOException { final byte[] content = new byte[length]; - file.read(content, 0, length); + new FileInputStream(file).read(content, 0, length); return content; } diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java index 7a32392f885e2..14ed2e294f7d2 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java @@ -39,8 +39,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; @@ -314,10 +314,9 @@ private static byte[] bytesOf(String str) { private static byte[] readFileContents(RefCountedFSOutputStream file) throws IOException { final byte[] content = new byte[MathUtils.checkedDownCast(file.getPos())]; - try (InputStream inputStream = file.getInputStream()) { - int bytesRead = inputStream.read(content, 0, content.length); // TODO: 10/2/18 see if closed in download - Assert.assertEquals(file.getPos(), bytesRead); - } + File inputFile = file.getInputFile(); + long bytesRead = new FileInputStream(inputFile).read(content, 0, MathUtils.checkedDownCast(inputFile.length())); + Assert.assertEquals(file.getPos(), bytesRead); return content; } diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3MultiPartUploader.java b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java similarity index 55% rename from flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3MultiPartUploader.java rename to flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java index f446f70e2a702..4d37ce0ba058d 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3MultiPartUploader.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java @@ -18,7 +18,7 @@ package org.apache.flink.fs.s3hadoop; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.flink.util.MathUtils; import com.amazonaws.SdkBaseException; @@ -35,24 +35,26 @@ import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.WriteOperationHelper; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; +import java.io.OutputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.flink.util.Preconditions.checkNotNull; /** - * An implementation of the {@link S3MultiPartUploader} for the Hadoop S3A filesystem. + * An implementation of the {@link S3AccessHelper} for the Hadoop S3A filesystem. */ -public class HadoopS3MultiPartUploader implements S3MultiPartUploader { +public class HadoopS3AccessHelper implements S3AccessHelper { private final S3AFileSystem s3a; - private final InternalWriteOperationHelper s3uploader; + private final InternalWriteOperationHelper s3accessHelper; - public HadoopS3MultiPartUploader(S3AFileSystem s3a, Configuration conf) { - this.s3uploader = new InternalWriteOperationHelper( + public HadoopS3AccessHelper(S3AFileSystem s3a, Configuration conf) { + this.s3accessHelper = new InternalWriteOperationHelper( checkNotNull(s3a), checkNotNull(conf) ); @@ -61,25 +63,58 @@ public HadoopS3MultiPartUploader(S3AFileSystem s3a, Configuration conf) { @Override public String startMultiPartUpload(String key) throws IOException { - return s3uploader.initiateMultiPartUpload(key); + return s3accessHelper.initiateMultiPartUpload(key); } @Override - public UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream inputStream, long length) throws IOException { - final UploadPartRequest uploadRequest = s3uploader.newUploadPartRequest( - key, uploadId, partNumber, MathUtils.checkedDownCast(length), inputStream, null, 0L); - return s3uploader.uploadPart(uploadRequest); + public UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException { + final UploadPartRequest uploadRequest = s3accessHelper.newUploadPartRequest( + key, uploadId, partNumber, MathUtils.checkedDownCast(length), null, inputFile, 0L); + return s3accessHelper.uploadPart(uploadRequest); } @Override - public PutObjectResult uploadIncompletePart(String key, InputStream inputStream, long length) throws IOException { - final PutObjectRequest putRequest = s3uploader.createPutObjectRequest(key, inputStream, length); - return s3uploader.putObject(putRequest); + public PutObjectResult putObject(String key, File inputFile) throws IOException { + final PutObjectRequest putRequest = s3accessHelper.createPutObjectRequest(key, inputFile); + return s3accessHelper.putObject(putRequest); } @Override public CompleteMultipartUploadResult commitMultiPartUpload(String destKey, String uploadId, List partETags, long length, AtomicInteger errorCount) throws IOException { - return s3uploader.completeMPUwithRetries(destKey, uploadId, partETags, length, errorCount); + return s3accessHelper.completeMPUwithRetries(destKey, uploadId, partETags, length, errorCount); + } + + @Override + public boolean deleteObject(String key) throws IOException { + return s3a.delete(new org.apache.hadoop.fs.Path('/' + key), false); + } + + @Override + public long getObject(String key, File targetLocation) throws IOException { + long numBytes = 0L; + try ( + final OutputStream outStream = new FileOutputStream(targetLocation); + final org.apache.hadoop.fs.FSDataInputStream inStream = + s3a.open(new org.apache.hadoop.fs.Path('/' + key)) + ) { + final byte[] buffer = new byte[32 * 1024]; + + int numRead; + while ((numRead = inStream.read(buffer)) != -1) { + outStream.write(buffer, 0, numRead); + numBytes += numRead; + } + } + + // some sanity checks + if (numBytes != targetLocation.length()) { + throw new IOException(String.format("Error recovering writer: " + + "Downloading the last data chunk file gives incorrect length. " + + "File=%d bytes, Stream=%d bytes", + targetLocation.length(), numBytes)); + } + + return numBytes; } @Override @@ -94,7 +129,7 @@ public ObjectMetadata getObjectMetadata(String key) throws IOException { /** * Internal {@link WriteOperationHelper} that is wrapped so that it only exposes - * the functionality we need for the {@link S3MultiPartUploader}. + * the functionality we need for the {@link S3AccessHelper}. */ private static final class InternalWriteOperationHelper extends WriteOperationHelper { diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java index 897629f791039..2637e7b2e23cb 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java @@ -21,7 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.fs.s3.common.AbstractS3FileSystemFactory; import org.apache.flink.fs.s3.common.HadoopConfigLoader; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem; @@ -96,8 +96,8 @@ else if (scheme != null && authority == null) { @Nullable @Override - protected S3MultiPartUploader getS3AccessHelper(FileSystem fs) { + protected S3AccessHelper getS3AccessHelper(FileSystem fs) { final S3AFileSystem s3Afs = (S3AFileSystem) fs; - return new HadoopS3MultiPartUploader(s3Afs, s3Afs.getConf()); + return new HadoopS3AccessHelper(s3Afs, s3Afs.getConf()); } } diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemBehaviorITCase.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemBehaviorITCase.java index c8aaaeef74d05..df73cdae34cb1 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemBehaviorITCase.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemBehaviorITCase.java @@ -23,9 +23,9 @@ import org.apache.flink.core.fs.FileSystemBehaviorTestSuite; import org.apache.flink.core.fs.FileSystemKind; import org.apache.flink.core.fs.Path; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.BeforeClass; import java.io.IOException; @@ -36,24 +36,17 @@ */ public class HadoopS3FileSystemBehaviorITCase extends FileSystemBehaviorTestSuite { - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); } @@ -69,7 +62,7 @@ public FileSystem getFileSystem() throws Exception { @Override public Path getBasePath() throws Exception { - return new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); + return new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR); } @Override diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java index 6dbdac511f4ac..70b5bce10a5b1 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java @@ -25,22 +25,18 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystem.WriteMode; import org.apache.flink.core.fs.Path; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.apache.flink.util.TestLogger; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; import java.util.UUID; import static org.apache.flink.core.fs.FileSystemTestUtils.checkPathEventualExistence; @@ -56,70 +52,45 @@ * consistency guarantees * and what the {@link org.apache.hadoop.fs.s3a.S3AFileSystem} offers. */ -@RunWith(Parameterized.class) public class HadoopS3FileSystemITCase extends TestLogger { - @Parameterized.Parameter - public String scheme; - - @Parameterized.Parameters(name = "Scheme = {0}") - public static List parameters() { - return Arrays.asList("s3", "s3a"); - } - - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - /** * Will be updated by {@link #checkCredentialsAndSetup()} if the test is not skipped. */ - private static boolean skipTest = true; + private static FileSystem fileSystem; + + private static Path basePath; @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); - // check for uniqueness of the test directory - final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); - final FileSystem fs = directory.getFileSystem(); + basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID()); + fileSystem = basePath.getFileSystem(); + // check for uniqueness of the test directory // directory must not yet exist - assertFalse(fs.exists(directory)); - - skipTest = false; + assertFalse(fileSystem.exists(basePath)); } @AfterClass public static void cleanUp() throws IOException, InterruptedException { try { - if (!skipTest) { + if (fileSystem != null) { final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - // initialize configuration with valid credentials - final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); - FileSystem.initialize(conf); - - final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); - final FileSystem fs = directory.getFileSystem(); // clean up - fs.delete(directory, true); + fileSystem.delete(basePath, true); // now directory must be gone - checkPathEventualExistence(fs, directory, false, deadline); + checkPathEventualExistence(fileSystem, basePath, false, deadline); } } finally { @@ -127,57 +98,12 @@ public static void cleanUp() throws IOException, InterruptedException { } } - private String getBasePath() { - return scheme + "://" + BUCKET + '/' + TEST_DATA_DIR + "/" + scheme; - } - - @Test - public void testConfigKeysForwarding() throws Exception { - final Path path = new Path(getBasePath()); - - // standard Hadoop-style credential keys - { - Configuration conf = new Configuration(); - conf.setString("fs.s3a.access.key", ACCESS_KEY); - conf.setString("fs.s3a.secret.key", SECRET_KEY); - - FileSystem.initialize(conf); - path.getFileSystem(); - } - - // shortened Hadoop-style credential keys - { - Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); - - FileSystem.initialize(conf); - path.getFileSystem(); - } - - // shortened Presto-style credential keys - { - Configuration conf = new Configuration(); - conf.setString("s3.access-key", ACCESS_KEY); - conf.setString("s3.secret-key", SECRET_KEY); - - FileSystem.initialize(conf); - path.getFileSystem(); - } - } - @Test public void testSimpleFileWriteAndRead() throws Exception { final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); - final String testLine = "Hello Upload!"; - FileSystem.initialize(conf); - - final Path path = new Path(getBasePath() + "/test.txt"); + final Path path = new Path(basePath, "test.txt"); final FileSystem fs = path.getFileSystem(); try { @@ -207,13 +133,8 @@ public void testSimpleFileWriteAndRead() throws Exception { @Test public void testDirectoryListing() throws Exception { final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); - - FileSystem.initialize(conf); - final Path directory = new Path(getBasePath() + "/testdir/"); + final Path directory = new Path(basePath, "testdir/"); final FileSystem fs = directory.getFileSystem(); // directory must not yet exist diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java index 6faf5b2575f7b..4471b3868dfdd 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java @@ -29,6 +29,7 @@ * Unit tests for the S3 file system support via Hadoop's {@link org.apache.hadoop.fs.s3a.S3AFileSystem}. */ public class HadoopS3FileSystemTest { + @Test public void testShadingOfAwsCredProviderConfig() { final Configuration conf = new Configuration(); @@ -41,4 +42,54 @@ public void testShadingOfAwsCredProviderConfig() { assertEquals("org.apache.flink.fs.s3hadoop.shaded.com.amazonaws.auth.ContainerCredentialsProvider", hadoopConfig.get("fs.s3a.aws.credentials.provider")); } + + // ------------------------------------------------------------------------ + // These tests check that the S3FileSystemFactory properly forwards + // various patterns of keys for credentials. + // ------------------------------------------------------------------------ + + /** + * Test forwarding of standard Hadoop-style credential keys. + */ + @Test + public void testConfigKeysForwardingHadoopStyle() { + Configuration conf = new Configuration(); + conf.setString("fs.s3a.access.key", "test_access_key"); + conf.setString("fs.s3a.secret.key", "test_secret_key"); + + checkHadoopAccessKeys(conf, "test_access_key", "test_secret_key"); + } + + /** + * Test forwarding of shortened Hadoop-style credential keys. + */ + @Test + public void testConfigKeysForwardingShortHadoopStyle() { + Configuration conf = new Configuration(); + conf.setString("s3.access.key", "my_key_a"); + conf.setString("s3.secret.key", "my_key_b"); + + checkHadoopAccessKeys(conf, "my_key_a", "my_key_b"); + } + + /** + * Test forwarding of shortened Presto-style credential keys. + */ + @Test + public void testConfigKeysForwardingPrestoStyle() { + Configuration conf = new Configuration(); + conf.setString("s3.access-key", "clé d'accès"); + conf.setString("s3.secret-key", "clef secrète"); + checkHadoopAccessKeys(conf, "clé d'accès", "clef secrète"); + } + + private static void checkHadoopAccessKeys(Configuration flinkConf, String accessKey, String secretKey) { + HadoopConfigLoader configLoader = S3FileSystemFactory.createHadoopConfigLoader(); + configLoader.setFlinkConfig(flinkConf); + + org.apache.hadoop.conf.Configuration hadoopConf = configLoader.getOrLoadHadoopConfig(); + + assertEquals(accessKey, hadoopConf.get("fs.s3a.access.key", null)); + assertEquals(secretKey, hadoopConf.get("fs.s3a.secret.key", null)); + } } diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemsSchemesTest.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemsSchemesTest.java new file mode 100644 index 0000000000000..a57b4da35e852 --- /dev/null +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemsSchemesTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.fs.s3hadoop; + +import org.apache.flink.core.fs.FileSystemFactory; + +import org.junit.Test; + +import java.util.ServiceLoader; + +import static org.junit.Assert.fail; + +/** + * This test validates that the S3 file system registers both under s3:// and s3a://. + */ +public class HadoopS3FileSystemsSchemesTest { + + @Test + public void testS3Factory() { + testFactory("s3"); + } + + @Test + public void testS3AFactory() { + testFactory("s3a"); + } + + private static void testFactory(String scheme) { + ServiceLoader serviceLoader = ServiceLoader.load(FileSystemFactory.class); + for (FileSystemFactory fs : serviceLoader) { + if (scheme.equals(fs.getScheme())) { + // found the matching scheme + return; + } + } + + fail("No factory available for scheme " + scheme); + } +} diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionTest.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionITCase.java similarity index 88% rename from flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionTest.java rename to flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionITCase.java index 634fa00344d84..59b8b752a7084 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionTest.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionITCase.java @@ -25,12 +25,12 @@ import org.apache.flink.core.fs.RecoverableFsDataOutputStream; import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.fs.s3.common.FlinkS3FileSystem; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.apache.flink.util.StringUtils; import org.apache.flink.util.TestLogger; import org.junit.After; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -51,14 +51,10 @@ * Tests for exception throwing in the * {@link org.apache.flink.fs.s3.common.writer.S3RecoverableWriter S3RecoverableWriter}. */ -public class HadoopS3RecoverableWriterExceptionTest extends TestLogger { +public class HadoopS3RecoverableWriterExceptionITCase extends TestLogger { // ----------------------- S3 general configuration ----------------------- - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final long PART_UPLOAD_MIN_SIZE_VALUE = 7L << 20; private static final int MAX_CONCURRENT_UPLOADS_VALUE = 2; @@ -66,9 +62,7 @@ public class HadoopS3RecoverableWriterExceptionTest extends TestLogger { private static final Random RND = new Random(); - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - - private static final Path basePath = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); + private static Path basePath; private static FlinkS3FileSystem fileSystem; @@ -89,14 +83,14 @@ public class HadoopS3RecoverableWriterExceptionTest extends TestLogger { @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); + + basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID()); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE); conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE); diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterTest.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterITCase.java similarity index 84% rename from flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterTest.java rename to flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterITCase.java index 4a1368a815e86..6c8619de0ed91 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterTest.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterITCase.java @@ -27,6 +27,8 @@ import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.fs.s3.common.FlinkS3FileSystem; +import org.apache.flink.fs.s3.common.writer.S3Recoverable; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.apache.flink.util.MathUtils; import org.apache.flink.util.StringUtils; import org.apache.flink.util.TestLogger; @@ -34,7 +36,6 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -42,6 +43,7 @@ import org.junit.rules.TemporaryFolder; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; @@ -59,14 +61,10 @@ /** * Tests for the {@link org.apache.flink.fs.s3.common.writer.S3RecoverableWriter S3RecoverableWriter}. */ -public class HadoopS3RecoverableWriterTest extends TestLogger { +public class HadoopS3RecoverableWriterITCase extends TestLogger { // ----------------------- S3 general configuration ----------------------- - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final long PART_UPLOAD_MIN_SIZE_VALUE = 7L << 20; private static final int MAX_CONCURRENT_UPLOADS_VALUE = 2; @@ -74,9 +72,7 @@ public class HadoopS3RecoverableWriterTest extends TestLogger { private static final Random RND = new Random(); - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - - private static final Path basePath = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); + private static Path basePath; private static FlinkS3FileSystem fileSystem; @@ -101,14 +97,14 @@ public class HadoopS3RecoverableWriterTest extends TestLogger { @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); + + basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID()); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE); conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE); @@ -219,6 +215,50 @@ public void testCommitAfterPersist() throws Exception { Assert.assertEquals(testData1 + testData2, getContentsOfFile(path)); } + @Test(expected = FileNotFoundException.class) + public void testCleanupRecoverableState() throws Exception { + final RecoverableWriter writer = getRecoverableWriter(); + final Path path = new Path(basePathForTest, "part-0"); + + final RecoverableFsDataOutputStream stream = writer.open(path); + stream.write(bytesOf(testData1)); + S3Recoverable recoverable = (S3Recoverable) stream.persist(); + + stream.closeForCommit().commit(); + + // still the data is there as we have not deleted them from the tmp object + final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName())); + Assert.assertEquals(testData1, content); + + boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable); + Assert.assertTrue(successfullyDeletedState); + + // this should throw the exception as we deleted the file. + getContentsOfFile(new Path('/' + recoverable.incompleteObjectName())); + } + + @Test + public void testCallingDeleteObjectTwiceDoesNotThroughException() throws Exception { + final RecoverableWriter writer = getRecoverableWriter(); + final Path path = new Path(basePathForTest, "part-0"); + + final RecoverableFsDataOutputStream stream = writer.open(path); + stream.write(bytesOf(testData1)); + S3Recoverable recoverable = (S3Recoverable) stream.persist(); + + stream.closeForCommit().commit(); + + // still the data is there as we have not deleted them from the tmp object + final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName())); + Assert.assertEquals(testData1, content); + + boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable); + Assert.assertTrue(successfullyDeletedState); + + boolean unsuccessfulDeletion = writer.cleanupRecoverableState(recoverable); + Assert.assertFalse(unsuccessfulDeletion); + } + // ----------------------- Test Recovery ----------------------- @Test diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml index cf35a04a585b2..6560c320976b0 100644 --- a/flink-filesystems/flink-s3-fs-presto/pom.xml +++ b/flink-filesystems/flink-s3-fs-presto/pom.xml @@ -269,6 +269,7 @@ under the License. META-INF/maven/io.airlift/** META-INF/maven/com*/** META-INF/maven/org.apache.flink/force-shading/** + META-INF/LICENSE.txt diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java index b579d6ebf0496..0fb28571960c0 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java @@ -21,7 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.fs.s3.common.AbstractS3FileSystemFactory; import org.apache.flink.fs.s3.common.HadoopConfigLoader; -import org.apache.flink.fs.s3.common.writer.S3MultiPartUploader; +import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.flink.util.FlinkRuntimeException; import com.facebook.presto.hive.PrestoS3FileSystem; @@ -92,7 +92,7 @@ else if (scheme != null && authority == null) { @Nullable @Override - protected S3MultiPartUploader getS3AccessHelper(FileSystem fs) { + protected S3AccessHelper getS3AccessHelper(FileSystem fs) { return null; } diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..b81e974b80e39 --- /dev/null +++ b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE @@ -0,0 +1,26 @@ +flink-s3-fs-presto +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.facebook.presto:presto-hive:0.185 +- com.facebook.presto.hadoop:hadoop-apache2:2.7.3-1 +- com.google.guava:guava:21.0 +- io.airlift:configuration:0.148 +- io.airlift:log:0.148 +- io.airlift:stats:0.148 +- io.airlift:units:1.0 +- io.airlift:slice:0.31 +- com.fasterxml.jackson.core:jackson-annotations:2.8.1 +- com.fasterxml.jackson.core:jackson-core:2.8.1 +- com.fasterxml.jackson.core:jackson-databind:2.8.1 +- joda-time:joda-time:2.5 +- org.weakref:jmxutils:1.19 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.9 diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-hdrhistogram b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-hdrhistogram new file mode 100644 index 0000000000000..09c38ea51608e --- /dev/null +++ b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemBehaviorITCase.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemBehaviorITCase.java index 812404ce63911..1603927749ef2 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemBehaviorITCase.java +++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemBehaviorITCase.java @@ -23,9 +23,9 @@ import org.apache.flink.core.fs.FileSystemBehaviorTestSuite; import org.apache.flink.core.fs.FileSystemKind; import org.apache.flink.core.fs.Path; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.BeforeClass; import java.io.IOException; @@ -36,24 +36,17 @@ */ public class PrestoS3FileSystemBehaviorITCase extends FileSystemBehaviorTestSuite { - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); } @@ -69,7 +62,7 @@ public FileSystem getFileSystem() throws Exception { @Override public Path getBasePath() throws Exception { - return new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); + return new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR); } @Override diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java index 07835740e2dc1..9a008a176d66a 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java +++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java @@ -25,17 +25,21 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystem.WriteMode; import org.apache.flink.core.fs.Path; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.apache.flink.util.TestLogger; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import static com.facebook.presto.hive.PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS; @@ -53,25 +57,27 @@ * consistency guarantees * and what the {@link com.facebook.presto.hive.PrestoS3FileSystem} offers. */ +@RunWith(Parameterized.class) public class PrestoS3FileSystemITCase extends TestLogger { - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); + @Parameterized.Parameter + public String scheme; - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); + @Parameterized.Parameters(name = "Scheme = {0}") + public static List parameters() { + return Arrays.asList("s3", "s3p"); + } - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); + private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); @BeforeClass public static void checkIfCredentialsArePresent() { - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); } @Test public void testConfigKeysForwarding() throws Exception { - final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); + final Path path = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR); // access without credentials should fail { @@ -90,8 +96,8 @@ public void testConfigKeysForwarding() throws Exception { { Configuration conf = new Configuration(); conf.setString(S3_USE_INSTANCE_CREDENTIALS, "false"); - conf.setString("presto.s3.access-key", ACCESS_KEY); - conf.setString("presto.s3.secret-key", SECRET_KEY); + conf.setString("presto.s3.access-key", S3TestCredentials.getS3AccessKey()); + conf.setString("presto.s3.secret-key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); path.getFileSystem().exists(path); @@ -101,8 +107,8 @@ public void testConfigKeysForwarding() throws Exception { { Configuration conf = new Configuration(); conf.setString(S3_USE_INSTANCE_CREDENTIALS, "false"); - conf.setString("s3.access-key", ACCESS_KEY); - conf.setString("s3.secret-key", SECRET_KEY); + conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); path.getFileSystem().exists(path); @@ -112,8 +118,8 @@ public void testConfigKeysForwarding() throws Exception { { Configuration conf = new Configuration(); conf.setString(S3_USE_INSTANCE_CREDENTIALS, "false"); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); path.getFileSystem().exists(path); @@ -123,8 +129,8 @@ public void testConfigKeysForwarding() throws Exception { { Configuration conf = new Configuration(); conf.setString(S3_USE_INSTANCE_CREDENTIALS, "false"); - conf.setString("presto.s3.access.key", ACCESS_KEY); - conf.setString("presto.s3.secret.key", SECRET_KEY); + conf.setString("presto.s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("presto.s3.secret.key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); path.getFileSystem().exists(path); @@ -138,14 +144,14 @@ public void testConfigKeysForwarding() throws Exception { public void testSimpleFileWriteAndRead() throws Exception { final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs final Configuration conf = new Configuration(); - conf.setString("s3.access-key", ACCESS_KEY); - conf.setString("s3.secret-key", SECRET_KEY); + conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey()); final String testLine = "Hello Upload!"; FileSystem.initialize(conf); - final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt"); + final Path path = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR + "/test.txt"); final FileSystem fs = path.getFileSystem(); try { @@ -173,12 +179,12 @@ public void testSimpleFileWriteAndRead() throws Exception { public void testDirectoryListing() throws Exception { final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs final Configuration conf = new Configuration(); - conf.setString("s3.access-key", ACCESS_KEY); - conf.setString("s3.secret-key", SECRET_KEY); + conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey()); FileSystem.initialize(conf); - final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/testdir/"); + final Path directory = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR + "/testdir/"); final FileSystem fs = directory.getFileSystem(); // directory must not yet exist diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3RecoverableWriterTest.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3RecoverableWriterTest.java index 580d957db2327..59d2f165e2e14 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3RecoverableWriterTest.java +++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3RecoverableWriterTest.java @@ -21,15 +21,15 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; import org.apache.flink.fs.s3.common.FlinkS3FileSystem; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; +import java.net.URI; import java.util.UUID; import static org.apache.flink.fs.s3.common.AbstractS3FileSystemFactory.MAX_CONCURRENT_UPLOADS; @@ -42,10 +42,6 @@ public class PrestoS3RecoverableWriterTest { // ----------------------- S3 general configuration ----------------------- - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final long PART_UPLOAD_MIN_SIZE_VALUE = 7L << 20; private static final int MAX_CONCURRENT_UPLOADS_VALUE = 2; @@ -53,21 +49,17 @@ public class PrestoS3RecoverableWriterTest { private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - private static final Path basePath = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR); - // ----------------------- Test Lifecycle ----------------------- @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); // initialize configuration with valid credentials final Configuration conf = new Configuration(); - conf.setString("s3.access.key", ACCESS_KEY); - conf.setString("s3.secret.key", SECRET_KEY); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE); conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE); @@ -87,7 +79,8 @@ public static void cleanUp() throws IOException { @Test(expected = UnsupportedOperationException.class) public void requestingRecoverableWriterShouldThroughException() throws Exception { - FlinkS3FileSystem fileSystem = (FlinkS3FileSystem) FileSystem.get(basePath.toUri()); + URI s3Uri = URI.create(S3TestCredentials.getTestBucketUri()); + FlinkS3FileSystem fileSystem = (FlinkS3FileSystem) FileSystem.get(s3Uri); fileSystem.createRecoverableWriter(); } } diff --git a/flink-filesystems/flink-swift-fs-hadoop/pom.xml b/flink-filesystems/flink-swift-fs-hadoop/pom.xml index d0445d659ceee..cd6ff253de823 100644 --- a/flink-filesystems/flink-swift-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-swift-fs-hadoop/pom.xml @@ -247,6 +247,9 @@ under the License. META-INF/maven/org.apache.commons/** META-INF/maven/org.apache.flink/flink-hadoop-fs/** META-INF/maven/org.apache.flink/force-shading/** + META-INF/LICENSE.txt + META-INF/ASL2.0 + META-INF/README.txt core-default.xml diff --git a/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..9f2c6350cf3be --- /dev/null +++ b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -0,0 +1,1655 @@ +flink-swift-fs-hadoop +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-annotations:2.8.1 +- org.apache.hadoop:hadoop-auth:2.8.1 +- org.apache.hadoop:hadoop-client:2.8.1 +- org.apache.hadoop:hadoop-common:2.8.1 +- org.apache.hadoop:hadoop-hdfs:2.8.1 +- org.apache.hadoop:hadoop-hdfs-client:2.8.1 +- org.apache.hadoop:hadoop-openstack:2.8.1 +- org.apache.htrace:htrace-core4:4.0.1-incubating +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- commons-beanutils:commons-beanutils:1.8.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- commons-httpclient:commons-httpclient:3.1 +- com.google.guava:guava:11.0.2 +- com.google.code.gson:gson:2.2.4 +- com.squareup.okhttp:okhttp:2.4.0 +- com.squareup.okio:okio:1.4.0 +- net.minidev:json-smart:1.1.1 +- com.nimbusds:nimbus-jose-jwt:3.9 +- org.mortbay.jetty:jetty-sslengine:6.1.26 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +This project bundles the following dependencies under the BSD License. +See bundled license files for details. + +- xmlenc:xmlenc:0.52 +- com.google.protobuf:protobuf-java:2.5.0 + +This project bundles "net.jcip:jcip-annotations:1.0". +Written by Brian Goetz and Tim Peierls with assistance from members of JCP JSR-166 Expert Group +and released to the public domain, as explained by the Creative Commons public domain license. +https://creativecommons.org/licenses/publicdomain/ + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles org.apache.hadoop:*:2.8.1 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +/** + * Copyright 2008,2009,2010 Massachusetts Institute of Technology. + * All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +And the binary distribution of this product bundles these dependencies under the +following license: +Mockito 1.8.5 +SLF4J 1.7.10 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +-------------------------------------------------------------------------------- + +Copyright jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +==== + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js.gz +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.0.0 +-------------------------------------------------------------------------------- +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +servlet-api 2.5 +jsp-api 2.1 +Streaming API for XML 1.0 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +Jersey 1.9 +JAXB API bundle for GlassFish V3 2.2.2 +JAXB RI 2.2.3 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2. License Grants. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +ecj-4.3.1.jar +-------------------------------------------------------------------------------- +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +ASM Core 3.2 +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +xmlenc Library 0.52 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +Java Concurrency in Practice book annotations 1.0 +-------------------------------------------------------------------------------- +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + diff --git a/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-protobuf b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-protobuf new file mode 100644 index 0000000000000..21645bec9d5d1 --- /dev/null +++ b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-protobuf @@ -0,0 +1,36 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + diff --git a/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-xmlenc b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-xmlenc new file mode 100644 index 0000000000000..a1256bdf6f8fd --- /dev/null +++ b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-xmlenc @@ -0,0 +1,28 @@ +Copyright 2003-2005, Ernst de Haan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-formats/flink-avro-confluent-registry/pom.xml b/flink-formats/flink-avro-confluent-registry/pom.xml index 9d818519672bd..9b93e3572ac64 100644 --- a/flink-formats/flink-avro-confluent-registry/pom.xml +++ b/flink-formats/flink-avro-confluent-registry/pom.xml @@ -78,10 +78,31 @@ under the License. shade + false + + + io.confluent:* + com.fasterxml.jackson.core:* + org.apache.zookeeper:zookeeper + com.101tec:zkclient + + - com.fasterxml.jackson.core - org.apache.flink.formats.avro.registry.confluent.shaded.com.fasterxml.jackson.core + com.fasterxml.jackson + org.apache.flink.formats.avro.registry.confluent.shaded.com.fasterxml.jackson + + + org.apache.zookeeper + org.apache.flink.formats.avro.registry.confluent.shaded.org.apache.zookeeper + + + org.apache.jute + org.apache.flink.formats.avro.registry.confluent.shaded.org.apache.jute + + + org.I0Itec.zkclient + org.apache.flink.formats.avro.registry.confluent.shaded.org.101tec diff --git a/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE b/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..24f2ff846578c --- /dev/null +++ b/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE @@ -0,0 +1,15 @@ +flink-avro-confluent-registry +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.101tec:zkclient:0.10 +- com.fasterxml.jackson.core:jackson-databind:2.8.4 +- com.fasterxml.jackson.core:jackson-annotations:2.8.0 +- com.fasterxml.jackson.core:jackson-core:2.8.4 +- io.confluent:common-utils:3.3.1 +- io.confluent:kafka-schema-registry-client:3.3.1 +- org.apache.zookeeper:zookeeper:3.4.10 diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroFactory.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroFactory.java index 0ca25bfaa04e4..9a8bdcba1ca2c 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroFactory.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroFactory.java @@ -86,9 +86,9 @@ static AvroFactory create(Class type, @Nullable Schema currentSchema, return fromReflective(type, cl, Optional.ofNullable(previousSchema)); } - static AvroFactory createFromTypeAndSchemaString(Class type, @Nullable String schemaString) { - Schema schema = (schemaString != null) ? new Schema.Parser().parse(schemaString) : null; - return create(type, schema, null); + @Nullable + static Schema parseSchemaString(@Nullable String schemaString) { + return (schemaString == null) ? null : new Schema.Parser().parse(schemaString); } @SuppressWarnings("OptionalUsedAsFieldOrParameterType") diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java index 2be660ad27be0..da51117984d95 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java @@ -32,12 +32,15 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DatumReader; import org.apache.avro.io.DatumWriter; -import org.apache.avro.reflect.Nullable; import org.apache.avro.specific.SpecificRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import java.io.IOException; +import java.io.ObjectInputStream; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -75,11 +78,9 @@ public class AvroSerializer extends TypeSerializer { // -------- configuration fields, serializable ----------- - /** The class of the type that is serialized by this serializer. - */ - private final Class type; - private final SerializableAvroSchema schema; - private final SerializableAvroSchema previousSchema; + @Nonnull private Class type; + @Nonnull private SerializableAvroSchema schema; + @Nonnull private SerializableAvroSchema previousSchema; // -------- runtime fields, non-serializable, lazily initialized ----------- @@ -127,10 +128,10 @@ public AvroSerializer(Class type, Schema schema) { * Creates a new AvroSerializer for the type indicated by the given class. */ @Internal - AvroSerializer(Class type, @Nullable SerializableAvroSchema newSchema, @Nullable SerializableAvroSchema previousSchema) { + AvroSerializer(Class type, SerializableAvroSchema newSchema, SerializableAvroSchema previousSchema) { this.type = checkNotNull(type); - this.schema = newSchema; - this.previousSchema = previousSchema; + this.schema = checkNotNull(newSchema); + this.previousSchema = checkNotNull(previousSchema); } /** @@ -144,6 +145,7 @@ public AvroSerializer(Class type, Class typeToInstantiate) { // ------------------------------------------------------------------------ + @Nonnull public Class getType() { return type; } @@ -381,4 +383,90 @@ public AvroSchemaSerializerConfigSnapshot() { } } + + // -------- backwards compatibility with 1.5, 1.6 ----------- + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + /* + Please see FLINK-11436 for details on why manual deserialization is required. + + During the release of Flink 1.7, the value of serialVersionUID was uptick to 2L (was 1L before) + And although the AvroSerializer (along with it's snapshot class) were migrated to the new serialization + abstraction (hence free from Java serialization), there were composite serializers that were not migrated + and were serialized with Java serialization. In case that one of the nested serializers were Avro we would + bump into deserialization exception due to a wrong serialVersionUID. Unfortunately it is not possible to revert + the serialVersionUID back to 1L, because users might have snapshots with 2L present already. + To overcome this we first need to make sure that the AvroSerializer is being Java deserialized with + FailureTolerantObjectInputStream, and then we determine the serialized layout by looking at the fields. + + From: https://docs.oracle.com/javase/8/docs/platform/serialization/spec/class.html#a5421 + ------------------------------------------------------------------------------------------------------------- + The descriptors for primitive typed fields are written first + sorted by field name followed by descriptors for the object typed fields sorted by field name. + The names are sorted using String.compareTo. + ------------------------------------------------------------------------------------------------------------- + + pre 1.6 field order: [type] + pre 1.7 field order: [schemaString, type] + post 1.7 field order: [previousSchema, schema, type] + + We would use the first field to distinguish between the three different layouts. + To complicate things even further in pre 1.7, the field @schemaString could be + null or a string, but, in post 1.7, the field @previousSchema was never set to null, therefore + we can use the first field to determine the version. + + this logic should stay here as long as we support Flink 1.6 (along with Java serialized + TypeSerializers) + */ + final Object firstField = in.readObject(); + + if (firstField == null) { + // first field can only be NULL in 1.6 (schemaString) + read16Layout(null, in); + } + else if (firstField instanceof String) { + // first field is a String only in 1.6 (schemaString) + read16Layout((String) firstField, in); + } + else if (firstField instanceof Class) { + // first field is a Class only in 1.5 (type) + @SuppressWarnings("unchecked") Class type = (Class) firstField; + read15Layout(type); + } + else if (firstField instanceof SerializableAvroSchema) { + readCurrentLayout((SerializableAvroSchema) firstField, in); + } + else { + throw new IllegalStateException("Failed to Java-Deserialize an AvroSerializer instance. " + + "Was expecting a first field to be either a String or SerializableAvroSchema, but got: " + + "" + firstField.getClass()); + } + } + + private void read15Layout(Class type) { + this.previousSchema = new SerializableAvroSchema(); + this.schema = new SerializableAvroSchema(); + this.type = type; + } + + @SuppressWarnings("unchecked") + private void read16Layout(@Nullable String schemaString, ObjectInputStream in) + throws IOException, ClassNotFoundException { + + Schema schema = AvroFactory.parseSchemaString(schemaString); + Class type = (Class) in.readObject(); + + this.previousSchema = new SerializableAvroSchema(); + this.schema = new SerializableAvroSchema(schema); + this.type = type; + } + + @SuppressWarnings("unchecked") + private void readCurrentLayout(SerializableAvroSchema previousSchema, ObjectInputStream in) + throws IOException, ClassNotFoundException { + + this.previousSchema = previousSchema; + this.schema = (SerializableAvroSchema) in.readObject(); + this.type = (Class) in.readObject(); + } } diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerMigrationTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerMigrationTest.java index a5f2a8e128f01..8b35a929f5a82 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerMigrationTest.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerMigrationTest.java @@ -18,15 +18,33 @@ package org.apache.flink.formats.avro.typeutils; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase; +import org.apache.flink.core.memory.DataInputDeserializer; import org.apache.flink.formats.avro.generated.Address; +import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; import org.apache.avro.generic.GenericRecord; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Arrays; +import java.util.Base64; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static junit.framework.TestCase.assertSame; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; /** * Tests migrations for {@link AvroSerializerSnapshot}. @@ -62,4 +80,173 @@ public static Collection testSpecifications() { ); } + // --------------------------------------------------------------------------------------------------------------- + // The following batch of tests are making sure that AvroSerializer class is able to be Java-Deserialized. + // see [FLINK-11436] for more information. + + // Once we drop support for versions that carried snapshots with Java-Deserialized serializers we can drop this + // batch of tests. + // --------------------------------------------------------------------------------------------------------------- + + @Test + public void javaDeserializeFromFlink_1_5_ReflectiveRecord() throws IOException { + final String avroSerializerBase64 = "AAAAAQAAAQis7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAABAgABTAAEdHlwZXQAEUxqYXZhL2xhbmcvQ2xhc3M7eHIANG9yZy5hcGFj\n" + + "aGUuZmxpbmsuYXBpLmNvbW1vbi50eXBldXRpbHMuVHlwZVNlcmlhbGl6ZXIAAAAAAAAAAQIAAHhwdnIA\n" + + "Tm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJvU2VyaWFsaXplck1pZ3Jh\n" + + "dGlvblRlc3QkU2ltcGxlUG9qbwAAAAAAAAAAAAAAeHA="; + + TypeSerializer serializer = javaDeserialize(avroSerializerBase64); + assertThat(serializer, instanceOf(AvroSerializer.class)); + + AvroSerializer avroSerializer = (AvroSerializer) javaDeserialize(avroSerializerBase64); + assertSame(avroSerializer.getType(), SimplePojo.class); + assertThat(avroSerializer.getAvroSchema(), notNullValue()); + } + + @Test + public void javaDeserializeFromFlink_1_5_SpecificRecord() throws IOException { + final String avroSerializerBase64 = "AAAAAQAAASOs7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAABAgABTAAEdHlwZXQAEUxqYXZhL2xhbmcvQ2xhc3M7eHIANG9yZy5hcGFj\n" + + "aGUuZmxpbmsuYXBpLmNvbW1vbi50eXBldXRpbHMuVHlwZVNlcmlhbGl6ZXIAAAAAAAAAAQIAAHhwdnIA\n" + + "L29yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLmdlbmVyYXRlZC5BZGRyZXNz7Paj+KjgQ2oMAAB4\n" + + "cgArb3JnLmFwYWNoZS5hdnJvLnNwZWNpZmljLlNwZWNpZmljUmVjb3JkQmFzZQKi+azGtzQdDAAAeHA="; + + TypeSerializer serializer = javaDeserialize(avroSerializerBase64); + assertThat(serializer, instanceOf(AvroSerializer.class)); + + AvroSerializer avroSerializer = (AvroSerializer) javaDeserialize(avroSerializerBase64); + assertSame(avroSerializer.getType(), Address.class); + assertThat(avroSerializer.getAvroSchema(), is(Address.SCHEMA$)); + } + + @Test + public void javaDeserializeFromFlink_1_6() throws IOException { + final String avroSerializer = "AAAAAQAAAUis7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAABAgACTAAMc2NoZW1hU3RyaW5ndAASTGphdmEvbGFuZy9TdHJpbmc7TAAE\n" + + "dHlwZXQAEUxqYXZhL2xhbmcvQ2xhc3M7eHIANG9yZy5hcGFjaGUuZmxpbmsuYXBpLmNvbW1vbi50eXBl\n" + + "dXRpbHMuVHlwZVNlcmlhbGl6ZXIAAAAAAAAAAQIAAHhwcHZyAC9vcmcuYXBhY2hlLmZsaW5rLmZvcm1h\n" + + "dHMuYXZyby5nZW5lcmF0ZWQuQWRkcmVzc+z2o/io4ENqDAAAeHIAK29yZy5hcGFjaGUuYXZyby5zcGVj\n" + + "aWZpYy5TcGVjaWZpY1JlY29yZEJhc2UCovmsxrc0HQwAAHhw"; + + TypeSerializer avro = javaDeserialize(avroSerializer); + + assertThat(avro, instanceOf(AvroSerializer.class)); + } + + @Test + public void javaDeserializeFromFlink_1_6_GenericRecord() throws IOException { + String avroSerializerBase64 = "AAAAAQAAAges7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAABAgACTAAMc2NoZW1hU3RyaW5ndAASTGphdmEvbGFuZy9TdHJpbmc7TAAE\n" + + "dHlwZXQAEUxqYXZhL2xhbmcvQ2xhc3M7eHIANG9yZy5hcGFjaGUuZmxpbmsuYXBpLmNvbW1vbi50eXBl\n" + + "dXRpbHMuVHlwZVNlcmlhbGl6ZXIAAAAAAAAAAQIAAHhwdAEBeyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6\n" + + "IkFkZHJlc3MiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmZsaW5rLmZvcm1hdHMuYXZyby5nZW5lcmF0\n" + + "ZWQiLCJmaWVsZHMiOlt7Im5hbWUiOiJudW0iLCJ0eXBlIjoiaW50In0seyJuYW1lIjoic3RyZWV0Iiwi\n" + + "dHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImNpdHkiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3Rh\n" + + "dGUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiemlwIiwidHlwZSI6InN0cmluZyJ9XX12cgAlb3Jn\n" + + "LmFwYWNoZS5hdnJvLmdlbmVyaWMuR2VuZXJpY1JlY29yZAAAAAAAAAAAAAAAeHA="; + + TypeSerializer serializer = javaDeserialize(avroSerializerBase64); + + AvroSerializer avroSerializer = (AvroSerializer) serializer; + assertSame(avroSerializer.getType(), GenericRecord.class); + assertThat(avroSerializer.getAvroSchema(), notNullValue()); + } + + @Test + public void javaDeserializeFromFlink_1_7() throws IOException { + String avroSerializerBase64 = "AAAAAQAAAeKs7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAACAgADTAAOcHJldmlvdXNTY2hlbWF0AEBMb3JnL2FwYWNoZS9mbGluay9m\n" + + "b3JtYXRzL2F2cm8vdHlwZXV0aWxzL1NlcmlhbGl6YWJsZUF2cm9TY2hlbWE7TAAGc2NoZW1hcQB+AAFM\n" + + "AAR0eXBldAARTGphdmEvbGFuZy9DbGFzczt4cgA0b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5\n" + + "cGV1dGlscy5UeXBlU2VyaWFsaXplcgAAAAAAAAABAgAAeHBzcgA+b3JnLmFwYWNoZS5mbGluay5mb3Jt\n" + + "YXRzLmF2cm8udHlwZXV0aWxzLlNlcmlhbGl6YWJsZUF2cm9TY2hlbWEAAAAAAAAAAQMAAHhwdwEAeHNx\n" + + "AH4ABXcBAHh2cgAvb3JnLmFwYWNoZS5mbGluay5mb3JtYXRzLmF2cm8uZ2VuZXJhdGVkLkFkZHJlc3Ps\n" + + "9qP4qOBDagwAAHhyACtvcmcuYXBhY2hlLmF2cm8uc3BlY2lmaWMuU3BlY2lmaWNSZWNvcmRCYXNlAqL5\n" + + "rMa3NB0MAAB4cA=="; + + AvroSerializer avroSerializer = (AvroSerializer) javaDeserialize(avroSerializerBase64); + assertSame(avroSerializer.getType(), Address.class); + assertThat(avroSerializer.getAvroSchema(), is(Address.SCHEMA$)); + } + + @Test + public void javaDeserializeFromFlink_1_7_afterInitialization() throws IOException { + String avroSerializerBase64 = "AAAAAQAAAeKs7QAFc3IANm9yZy5hcGFjaGUuZmxpbmsuZm9ybWF0cy5hdnJvLnR5cGV1dGlscy5BdnJv\n" + + "U2VyaWFsaXplcgAAAAAAAAACAgADTAAOcHJldmlvdXNTY2hlbWF0AEBMb3JnL2FwYWNoZS9mbGluay9m\n" + + "b3JtYXRzL2F2cm8vdHlwZXV0aWxzL1NlcmlhbGl6YWJsZUF2cm9TY2hlbWE7TAAGc2NoZW1hcQB+AAFM\n" + + "AAR0eXBldAARTGphdmEvbGFuZy9DbGFzczt4cgA0b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5\n" + + "cGV1dGlscy5UeXBlU2VyaWFsaXplcgAAAAAAAAABAgAAeHBzcgA+b3JnLmFwYWNoZS5mbGluay5mb3Jt\n" + + "YXRzLmF2cm8udHlwZXV0aWxzLlNlcmlhbGl6YWJsZUF2cm9TY2hlbWEAAAAAAAAAAQMAAHhwdwEAeHNx\n" + + "AH4ABXcBAHh2cgAvb3JnLmFwYWNoZS5mbGluay5mb3JtYXRzLmF2cm8uZ2VuZXJhdGVkLkFkZHJlc3Ps\n" + + "9qP4qOBDagwAAHhyACtvcmcuYXBhY2hlLmF2cm8uc3BlY2lmaWMuU3BlY2lmaWNSZWNvcmRCYXNlAqL5\n" + + "rMa3NB0MAAB4cA=="; + + AvroSerializer avroSerializer = (AvroSerializer) javaDeserialize(avroSerializerBase64); + assertSame(avroSerializer.getType(), Address.class); + assertThat(avroSerializer.getAvroSchema(), is(Address.SCHEMA$)); + } + + @Test + public void compositeSerializerFromFlink_1_6_WithNestedAvroSerializer() throws IOException { + String streamElementSerializerBase64 = "AAAAAQAAAq2s7QAFc3IAR29yZy5hcGFjaGUuZmxpbmsuc3RyZWFtaW5nLnJ1bnRpbWUuc3RyZWFtcmVj\n" + + "b3JkLlN0cmVhbUVsZW1lbnRTZXJpYWxpemVyAAAAAAAAAAECAAFMAA50eXBlU2VyaWFsaXplcnQANkxv\n" + + "cmcvYXBhY2hlL2ZsaW5rL2FwaS9jb21tb24vdHlwZXV0aWxzL1R5cGVTZXJpYWxpemVyO3hyADRvcmcu\n" + + "YXBhY2hlLmZsaW5rLmFwaS5jb21tb24udHlwZXV0aWxzLlR5cGVTZXJpYWxpemVyAAAAAAAAAAECAAB4\n" + + "cHNyADZvcmcuYXBhY2hlLmZsaW5rLmZvcm1hdHMuYXZyby50eXBldXRpbHMuQXZyb1NlcmlhbGl6ZXIA\n" + + "AAAAAAAAAQIAAkwADHNjaGVtYVN0cmluZ3QAEkxqYXZhL2xhbmcvU3RyaW5nO0wABHR5cGV0ABFMamF2\n" + + "YS9sYW5nL0NsYXNzO3hxAH4AAnQBAXsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRyZXNzIiwibmFt\n" + + "ZXNwYWNlIjoib3JnLmFwYWNoZS5mbGluay5mb3JtYXRzLmF2cm8uZ2VuZXJhdGVkIiwiZmllbGRzIjpb\n" + + "eyJuYW1lIjoibnVtIiwidHlwZSI6ImludCJ9LHsibmFtZSI6InN0cmVldCIsInR5cGUiOiJzdHJpbmci\n" + + "fSx7Im5hbWUiOiJjaXR5IiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6InN0YXRlIiwidHlwZSI6InN0\n" + + "cmluZyJ9LHsibmFtZSI6InppcCIsInR5cGUiOiJzdHJpbmcifV19dnIAJW9yZy5hcGFjaGUuYXZyby5n\n" + + "ZW5lcmljLkdlbmVyaWNSZWNvcmQAAAAAAAAAAAAAAHhw"; + + StreamElementSerializer ser = (StreamElementSerializer) javaDeserialize(streamElementSerializerBase64); + TypeSerializer containedTypeSerializer = ser.getContainedTypeSerializer(); + + assertThat(containedTypeSerializer, instanceOf(AvroSerializer.class)); + + AvroSerializer avroSerializer = (AvroSerializer) containedTypeSerializer; + assertSame(avroSerializer.getType(), GenericRecord.class); + assertThat(avroSerializer.getAvroSchema(), is(Address.SCHEMA$)); + } + + @Test + public void makeSureThatFieldsWereNotChanged() { + // This test should be removed once we completely migrate all the composite serializers. + + List serializedFieldNames = Arrays.stream(AvroSerializer.class.getDeclaredFields()) + .filter(field -> !Modifier.isTransient(field.getModifiers())) + .filter(field -> !Modifier.isStatic(field.getModifiers())) + .map(Field::getName) + .sorted() + .collect(Collectors.toList()); + + assertThat(serializedFieldNames, is(asList("previousSchema", "schema", "type"))); + } + + @SuppressWarnings("deprecation") + private static TypeSerializer javaDeserialize(String base64) throws IOException { + byte[] bytes = Base64.getMimeDecoder().decode(base64); + DataInputDeserializer in = new DataInputDeserializer(bytes); + return TypeSerializerSerializationUtil.tryReadSerializer(in, Thread.currentThread().getContextClassLoader()); + } + + /** + * A simple pojo used in these tests. + */ + public static class SimplePojo { + private String foo; + + @SuppressWarnings("unused") + public String getFoo() { + return foo; + } + + @SuppressWarnings("unused") + public void setFoo(String foo) { + this.foo = foo; + } + } } diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java index d1f0a5f57787f..484ef860e2d37 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java +++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java @@ -19,13 +19,14 @@ package org.apache.flink.formats.parquet.avro; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo; import org.apache.flink.formats.parquet.generated.Address; -import org.apache.flink.formats.parquet.testutils.FiniteTestSource; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; +import org.apache.flink.streaming.util.FiniteTestSource; import org.apache.flink.test.util.AbstractTestBase; import org.apache.avro.Schema; @@ -156,11 +157,14 @@ private static void validateResults(File folder, GenericData dataModel, List File[] partFiles = buckets[0].listFiles(); assertNotNull(partFiles); - assertEquals(1, partFiles.length); - assertTrue(partFiles[0].length() > 0); + assertEquals(2, partFiles.length); - List

    results = readParquetFile(partFiles[0], dataModel); - assertEquals(expected, results); + for (File partFile : partFiles) { + assertTrue(partFile.length() > 0); + + final List> fileContent = readParquetFile(partFile, dataModel); + assertEquals(expected, fileContent); + } } private static List readParquetFile(File file, GenericData dataModel) throws IOException { diff --git a/flink-formats/flink-sequence-file/pom.xml b/flink-formats/flink-sequence-file/pom.xml new file mode 100644 index 0000000000000..be7c0d750a779 --- /dev/null +++ b/flink-formats/flink-sequence-file/pom.xml @@ -0,0 +1,100 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-formats + 1.7-SNAPSHOT + .. + + + flink-sequence-file + flink-sequence-file + + jar + + + + + org.apache.flink + flink-core + ${project.version} + provided + + + + + org.apache.flink + flink-shaded-hadoop2 + ${project.version} + provided + + + + + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${project.version} + test + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + test + + + + org.apache.flink + flink-hadoop-compatibility_${scala.binary.version} + ${project.version} + test + + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + dependency-convergence + + enforce + + + true + + + + + + + diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java new file mode 100644 index 0000000000000..169aa4ba7cbc9 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.sequencefile; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.BulkWriter; +import org.apache.flink.api.java.tuple.Tuple2; + +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Writable; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A {@link BulkWriter} implementation that wraps a {@link SequenceFile.Writer}. + * + * @param The type of key written. + * @param The type of value written. + */ +@PublicEvolving +public class SequenceFileWriter implements BulkWriter> { + + private final SequenceFile.Writer writer; + + SequenceFileWriter(SequenceFile.Writer writer) { + this.writer = checkNotNull(writer); + } + + @Override + public void addElement(Tuple2 element) throws IOException { + writer.append(element.f0, element.f1); + } + + @Override + public void flush() throws IOException { + writer.hsync(); + } + + @Override + public void finish() throws IOException { + writer.close(); + } +} diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java new file mode 100644 index 0000000000000..d7b96f6700271 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.sequencefile; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.BulkWriter; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.core.fs.FSDataOutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A factory that creates a SequenceFile {@link BulkWriter}. + * + * @param The type of key to write. It should be writable. + * @param The type of value to write. It should be writable. + */ +@PublicEvolving +public class SequenceFileWriterFactory implements BulkWriter.Factory> { + + private static final long serialVersionUID = 1L; + + /** A constant specifying that no compression is requested. */ + public static final String NO_COMPRESSION = "NO_COMPRESSION"; + + private final SerializableHadoopConfiguration serializableHadoopConfig; + private final Class keyClass; + private final Class valueClass; + private final String compressionCodecName; + private final SequenceFile.CompressionType compressionType; + + /** + * Creates a new SequenceFileWriterFactory using the given builder to assemble the + * SequenceFileWriter. + * + * @param hadoopConf The Hadoop configuration for Sequence File Writer. + * @param keyClass The class of key to write. + * @param valueClass The class of value to write. + */ + public SequenceFileWriterFactory(Configuration hadoopConf, Class keyClass, Class valueClass) { + this(hadoopConf, keyClass, valueClass, NO_COMPRESSION, SequenceFile.CompressionType.BLOCK); + } + + /** + * Creates a new SequenceFileWriterFactory using the given builder to assemble the + * SequenceFileWriter. + * + * @param hadoopConf The Hadoop configuration for Sequence File Writer. + * @param keyClass The class of key to write. + * @param valueClass The class of value to write. + * @param compressionCodecName The name of compression codec. + */ + public SequenceFileWriterFactory(Configuration hadoopConf, Class keyClass, Class valueClass, String compressionCodecName) { + this(hadoopConf, keyClass, valueClass, compressionCodecName, SequenceFile.CompressionType.BLOCK); + } + + /** + * Creates a new SequenceFileWriterFactory using the given builder to assemble the + * SequenceFileWriter. + * + * @param hadoopConf The Hadoop configuration for Sequence File Writer. + * @param keyClass The class of key to write. + * @param valueClass The class of value to write. + * @param compressionCodecName The name of compression codec. + * @param compressionType The type of compression level. + */ + public SequenceFileWriterFactory(Configuration hadoopConf, Class keyClass, Class valueClass, String compressionCodecName, SequenceFile.CompressionType compressionType) { + this.serializableHadoopConfig = new SerializableHadoopConfiguration(checkNotNull(hadoopConf)); + this.keyClass = checkNotNull(keyClass); + this.valueClass = checkNotNull(valueClass); + this.compressionCodecName = checkNotNull(compressionCodecName); + this.compressionType = checkNotNull(compressionType); + } + + @Override + public SequenceFileWriter create(FSDataOutputStream out) throws IOException { + org.apache.hadoop.fs.FSDataOutputStream stream = new org.apache.hadoop.fs.FSDataOutputStream(out, null); + CompressionCodec compressionCodec = getCompressionCodec(serializableHadoopConfig.get(), compressionCodecName); + SequenceFile.Writer writer = SequenceFile.createWriter( + serializableHadoopConfig.get(), + SequenceFile.Writer.stream(stream), + SequenceFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass), + SequenceFile.Writer.compression(compressionType, compressionCodec)); + return new SequenceFileWriter<>(writer); + } + + private CompressionCodec getCompressionCodec(Configuration conf, String compressionCodecName) { + checkNotNull(conf); + checkNotNull(compressionCodecName); + + if (compressionCodecName.equals(NO_COMPRESSION)) { + return null; + } + + CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); + CompressionCodec codec = codecFactory.getCodecByName(compressionCodecName); + if (codec == null) { + throw new RuntimeException("Codec " + compressionCodecName + " not found."); + } + return codec; + } +} + diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java new file mode 100644 index 0000000000000..8e00e072b69d1 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.sequencefile; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +/** + * Wrapper class for serialization of {@link Configuration}. + */ +class SerializableHadoopConfiguration implements Serializable { + + private static final long serialVersionUID = 1L; + + private transient Configuration hadoopConfig; + + SerializableHadoopConfiguration(Configuration hadoopConfig) { + this.hadoopConfig = hadoopConfig; + } + + Configuration get() { + return this.hadoopConfig; + } + + // -------------------- + private void writeObject(ObjectOutputStream out) throws IOException { + this.hadoopConfig.write(out); + } + + private void readObject(ObjectInputStream in) throws IOException { + final Configuration config = new Configuration(); + config.readFields(in); + + if (this.hadoopConfig == null) { + this.hadoopConfig = config; + } + } +} diff --git a/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java new file mode 100644 index 0000000000000..2b3f325579e92 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.sequencefile; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeHint; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; +import org.apache.flink.streaming.util.FiniteTestSource; +import org.apache.flink.test.util.AbstractTestBase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Integration test case for writing bulk encoded files with the + * {@link StreamingFileSink} with SequenceFile. + */ +public class SequenceStreamingFileSinkITCase extends AbstractTestBase { + + private final Configuration configuration = new Configuration(); + + private final List> testData = Arrays.asList( + new Tuple2<>(1L, "a"), + new Tuple2<>(2L, "b"), + new Tuple2<>(3L, "c") + ); + + @Test + public void testWriteSequenceFile() throws Exception { + final File folder = TEMPORARY_FOLDER.newFolder(); + final Path testPath = Path.fromLocalFile(folder); + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + env.enableCheckpointing(100); + + DataStream> stream = env.addSource( + new FiniteTestSource<>(testData), + TypeInformation.of(new TypeHint>() { + + }) + ); + + stream.map(new MapFunction, Tuple2>() { + @Override + public Tuple2 map(Tuple2 value) throws Exception { + return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1)); + } + }).addSink( + StreamingFileSink.forBulkFormat( + testPath, + new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2") + ).build()); + + env.execute(); + + validateResults(folder, testData); + } + + private List> readSequenceFile(File file) throws IOException { + SequenceFile.Reader reader = new SequenceFile.Reader( + configuration, SequenceFile.Reader.file(new org.apache.hadoop.fs.Path(file.toURI()))); + LongWritable key = new LongWritable(); + Text val = new Text(); + ArrayList> results = new ArrayList<>(); + while (reader.next(key, val)) { + results.add(new Tuple2<>(key.get(), val.toString())); + } + reader.close(); + return results; + } + + private void validateResults(File folder, List> expected) throws Exception { + File[] buckets = folder.listFiles(); + assertNotNull(buckets); + assertEquals(1, buckets.length); + + final File[] partFiles = buckets[0].listFiles(); + assertNotNull(partFiles); + assertEquals(2, partFiles.length); + + for (File partFile : partFiles) { + assertTrue(partFile.length() > 0); + + final List> fileContent = readSequenceFile(partFile); + assertEquals(expected, fileContent); + } + } +} diff --git a/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java new file mode 100644 index 0000000000000..ea0fb955dc5a4 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.sequencefile; + +import org.apache.hadoop.conf.Configuration; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +/** + * Tests for the {@link SerializableHadoopConfiguration}. + */ +public class SerializableHadoopConfigurationTest { + + private static final String TEST_KEY = "test-key"; + + private static final String TEST_VALUE = "test-value"; + + private Configuration configuration; + + @Before + public void createConfigWithCustomProperty() { + this.configuration = new Configuration(); + configuration.set(TEST_KEY, TEST_VALUE); + } + + @Test + public void customPropertiesSurviveSerializationDeserialization() throws IOException, ClassNotFoundException { + final SerializableHadoopConfiguration serializableConfigUnderTest = new SerializableHadoopConfiguration(configuration); + final byte[] serializedConfigUnderTest = serializeAndGetBytes(serializableConfigUnderTest); + final SerializableHadoopConfiguration deserializableConfigUnderTest = deserializeAndGetConfiguration(serializedConfigUnderTest); + + Assert.assertThat(deserializableConfigUnderTest.get(), hasTheSamePropertiesAs(configuration)); + } + + // ---------------------------------------- Matchers ---------------------------------------- // + + private static TypeSafeMatcher hasTheSamePropertiesAs(final Configuration expectedConfig) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(Configuration actualConfig) { + final String value = actualConfig.get(TEST_KEY); + return actualConfig != expectedConfig && value != null && expectedConfig.get(TEST_KEY).equals(value); + } + + @Override + public void describeTo(Description description) { + description.appendText("a Hadoop Configuration with property: key=") + .appendValue(TEST_KEY) + .appendText(" and value=") + .appendValue(TEST_VALUE); + } + }; + } + + // ---------------------------------------- Helper Methods ---------------------------------------- // + + private byte[] serializeAndGetBytes(SerializableHadoopConfiguration serializableConfigUnderTest) throws IOException { + try ( + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(byteStream) + ) { + out.writeObject(serializableConfigUnderTest); + out.flush(); + return byteStream.toByteArray(); + } + } + + private SerializableHadoopConfiguration deserializeAndGetConfiguration(byte[] serializedConfig) throws IOException, ClassNotFoundException { + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(serializedConfig))) { + return (SerializableHadoopConfiguration) in.readObject(); + } + } +} diff --git a/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties b/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000..644b884d04d91 --- /dev/null +++ b/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=OFF, testlogger +log4j.appender.testlogger=org.apache.log4j.ConsoleAppender +log4j.appender.testlogger.target=System.err +log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout +log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml index c17ca58831e26..4b0d2f6133bfb 100644 --- a/flink-formats/pom.xml +++ b/flink-formats/pom.xml @@ -40,6 +40,7 @@ under the License. flink-json flink-avro-confluent-registry flink-parquet + flink-sequence-file + + org.apache.flink + flink-core + ${project.version} + test-jar + test + + org.apache.flink flink-test-utils_${scala.binary.version} diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java index a2eedaf138c6c..7afbc508230c7 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java @@ -18,6 +18,7 @@ package org.apache.flink.cep.nfa.sharedbuffer; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -25,7 +26,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.IntSerializer; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -109,8 +109,10 @@ public boolean isImmutableType() { } @Override - public TypeSerializer> duplicate() { - return new LockableTypeSerializer<>(elementSerializer); + public LockableTypeSerializer duplicate() { + TypeSerializer elementSerializerCopy = elementSerializer.duplicate(); + return elementSerializerCopy == elementSerializer ? + this : new LockableTypeSerializer<>(elementSerializerCopy); } @Override @@ -120,7 +122,7 @@ public Lockable createInstance() { @Override public Lockable copy(Lockable from) { - return new Lockable(elementSerializer.copy(from.element), from.refCounter); + return new Lockable<>(elementSerializer.copy(from.element), from.refCounter); } @Override @@ -184,40 +186,30 @@ public boolean canEqual(Object obj) { } @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new LockableSerializerConfigSnapshot<>(elementSerializer); + public TypeSerializerSnapshot> snapshotConfiguration() { + return new LockableTypeSerializerSnapshot<>(elementSerializer); } + /** + * This cannot be removed until {@link TypeSerializerConfigSnapshot} is no longer supported. + */ @Override public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof LockableSerializerConfigSnapshot) { - @SuppressWarnings("unchecked") - LockableSerializerConfigSnapshot snapshot = (LockableSerializerConfigSnapshot) configSnapshot; - - Tuple2, TypeSerializerSnapshot> nestedSerializerAndConfig = - snapshot.getSingleNestedSerializerAndConfig(); - - CompatibilityResult inputCompatibilityResult = CompatibilityUtil.resolveCompatibilityResult( - nestedSerializerAndConfig.f0, - UnloadableDummyTypeSerializer.class, - nestedSerializerAndConfig.f1, - elementSerializer); - - return (inputCompatibilityResult.isRequiresMigration()) - ? CompatibilityResult.requiresMigration() - : CompatibilityResult.compatible(); - } else { - // backwards compatibility path - CompatibilityResult inputCompatibilityResult = CompatibilityUtil.resolveCompatibilityResult( - configSnapshot.restoreSerializer(), - UnloadableDummyTypeSerializer.class, - configSnapshot, - elementSerializer); - - return (inputCompatibilityResult.isRequiresMigration()) - ? CompatibilityResult.requiresMigration() - : CompatibilityResult.compatible(); - } + // backwards compatibility path + CompatibilityResult inputCompatibilityResult = CompatibilityUtil.resolveCompatibilityResult( + configSnapshot.restoreSerializer(), + UnloadableDummyTypeSerializer.class, + configSnapshot, + elementSerializer); + + return (inputCompatibilityResult.isRequiresMigration()) + ? CompatibilityResult.requiresMigration() + : CompatibilityResult.compatible(); + } + + @VisibleForTesting + TypeSerializer getElementSerializer() { + return elementSerializer; } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java deleted file mode 100644 index 9e78bc013ef2a..0000000000000 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.cep.nfa.sharedbuffer; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; - -/** - * A {@link TypeSerializerConfigSnapshot} for the {@link Lockable.LockableTypeSerializer}. - */ -@Internal -public class LockableSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - - private static final int VERSION = 1; - - /** This empty nullary constructor is required for deserializing the configuration. */ - public LockableSerializerConfigSnapshot() {} - - public LockableSerializerConfigSnapshot(TypeSerializer elementSerializer) { - super(elementSerializer); - } - - @Override - public int getVersion() { - return VERSION; - } - -} diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java new file mode 100644 index 0000000000000..44a4670cc1da8 --- /dev/null +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.nfa.sharedbuffer; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A {@link TypeSerializerSnapshot} for the {@link Lockable.LockableTypeSerializer}. + */ +@Internal +public class LockableTypeSerializerSnapshot implements TypeSerializerSnapshot> { + + private static final int CURRENT_VERSION = 1; + + private CompositeSerializerSnapshot nestedElementSerializerSnapshot; + + /** + * Constructor for read instantiation. + */ + public LockableTypeSerializerSnapshot() {} + + /** + * Constructor to create the snapshot for writing. + */ + public LockableTypeSerializerSnapshot(TypeSerializer elementSerializer) { + this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(elementSerializer)); + } + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public TypeSerializer> restoreSerializer() { + return new Lockable.LockableTypeSerializer<>(nestedElementSerializerSnapshot.getRestoreSerializer(0)); + } + + @Override + public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { + checkState(nestedElementSerializerSnapshot != null); + + if (newSerializer instanceof Lockable.LockableTypeSerializer) { + Lockable.LockableTypeSerializer serializer = (Lockable.LockableTypeSerializer) newSerializer; + + return nestedElementSerializerSnapshot.resolveCompatibilityWithNested( + TypeSerializerSchemaCompatibility.compatibleAsIs(), + serializer.getElementSerializer()); + } + else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + nestedElementSerializerSnapshot.writeCompositeSnapshot(out); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.nestedElementSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + } + +} diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshotMigrationTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshotMigrationTest.java new file mode 100644 index 0000000000000..bb3b7f27e108b --- /dev/null +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshotMigrationTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.nfa.sharedbuffer; + +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase; +import org.apache.flink.api.common.typeutils.base.StringSerializer; + +/** + * Migration test for the {@link LockableTypeSerializerSnapshot}. + */ +public class LockableTypeSerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase> { + + private static final String DATA = "flink-1.6-lockable-type-serializer-data"; + private static final String SNAPSHOT = "flink-1.6-lockable-type-serializer-snapshot"; + + public LockableTypeSerializerSnapshotMigrationTest() { + super( + TestSpecification.>builder( + "1.6-lockable-type-serializer", + Lockable.LockableTypeSerializer.class, + LockableTypeSerializerSnapshot.class) + .withSerializerProvider(() -> new Lockable.LockableTypeSerializer<>(StringSerializer.INSTANCE)) + .withSnapshotDataLocation(SNAPSHOT) + .withTestData(DATA, 10) + ); + } +} diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerTest.java new file mode 100644 index 0000000000000..a7aeb69ff6adc --- /dev/null +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.nfa.sharedbuffer; + +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.runtime.state.heap.TestDuplicateSerializer; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for {@link org.apache.flink.cep.nfa.sharedbuffer.Lockable.LockableTypeSerializer}. + */ +public class LockableTypeSerializerTest { + + /** + * This tests that {@link Lockable.LockableTypeSerializer#duplicate()} works as expected. + */ + @Test + public void testDuplicate() { + IntSerializer nonDuplicatingInnerSerializer = IntSerializer.INSTANCE; + Assert.assertSame(nonDuplicatingInnerSerializer, nonDuplicatingInnerSerializer.duplicate()); + Lockable.LockableTypeSerializer candidateTestShallowDuplicate = + new Lockable.LockableTypeSerializer<>(nonDuplicatingInnerSerializer); + Assert.assertSame(candidateTestShallowDuplicate, candidateTestShallowDuplicate.duplicate()); + + TestDuplicateSerializer duplicatingInnerSerializer = new TestDuplicateSerializer(); + Assert.assertNotSame(duplicatingInnerSerializer, duplicatingInnerSerializer.duplicate()); + + Lockable.LockableTypeSerializer candidateTestDeepDuplicate = + new Lockable.LockableTypeSerializer<>(duplicatingInnerSerializer); + + Lockable.LockableTypeSerializer deepDuplicate = candidateTestDeepDuplicate.duplicate(); + Assert.assertNotSame(candidateTestDeepDuplicate, deepDuplicate); + Assert.assertNotSame(candidateTestDeepDuplicate.getElementSerializer(), deepDuplicate.getElementSerializer()); + } +} diff --git a/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-data b/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-data new file mode 100644 index 0000000000000..b17504031a645 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-snapshot new file mode 100644 index 0000000000000..3d9fa0c07fbc3 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/flink-1.6-lockable-type-serializer-snapshot differ diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ByteValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ByteValueArraySerializerTest.java index 4a29318169a56..2f8f8fcf70973 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ByteValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ByteValueArraySerializerTest.java @@ -19,7 +19,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.ByteValue; @@ -28,7 +27,7 @@ /** * A test for the {@link ByteValueArraySerializer}. */ -public class ByteValueArraySerializerTest extends SerializerTestBase { +public class ByteValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/CharValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/CharValueArraySerializerTest.java index 86ee10dbf0d05..432f146bf9940 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/CharValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/CharValueArraySerializerTest.java @@ -19,7 +19,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.CharValue; @@ -28,7 +27,7 @@ /** * A test for the {@link CharValueArraySerializer}. */ -public class CharValueArraySerializerTest extends SerializerTestBase { +public class CharValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/DoubleValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/DoubleValueArraySerializerTest.java index 49c1f65cb2194..f0396d091ae21 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/DoubleValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/DoubleValueArraySerializerTest.java @@ -19,7 +19,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.DoubleValue; @@ -28,7 +27,7 @@ /** * A test for the {@link DoubleValueArraySerializer}. */ -public class DoubleValueArraySerializerTest extends SerializerTestBase { +public class DoubleValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/FloatValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/FloatValueArraySerializerTest.java index 14312c3550604..64a58fe8a6e25 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/FloatValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/FloatValueArraySerializerTest.java @@ -19,7 +19,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.FloatValue; @@ -28,7 +27,7 @@ /** * A test for the {@link FloatValueArraySerializer}. */ -public class FloatValueArraySerializerTest extends SerializerTestBase { +public class FloatValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializerTest.java index 88a367c4e01fb..21a9bcb29c06c 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializerTest.java @@ -18,7 +18,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.IntValue; @@ -27,7 +26,7 @@ /** * A test for the {@link IntValueArraySerializer}. */ -public class IntValueArraySerializerTest extends SerializerTestBase { +public class IntValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializerTest.java index 50201daf6c7a1..e617b57143170 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializerTest.java @@ -18,7 +18,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.LongValue; @@ -27,7 +26,7 @@ /** * A test for the {@link LongValueArraySerializer}. */ -public class LongValueArraySerializerTest extends SerializerTestBase { +public class LongValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/NullValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/NullValueArraySerializerTest.java index 8cf1efa37308a..8b30cddd185d6 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/NullValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/NullValueArraySerializerTest.java @@ -18,14 +18,13 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.NullValue; /** * A test for the {@link NullValueArraySerializer}. */ -public class NullValueArraySerializerTest extends SerializerTestBase { +public class NullValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ShortValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ShortValueArraySerializerTest.java index 005f4d1e254f5..ad5ce8b551cef 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ShortValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ShortValueArraySerializerTest.java @@ -19,7 +19,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.ShortValue; @@ -28,7 +27,7 @@ /** * A test for the {@link ShortValueArraySerializer}. */ -public class ShortValueArraySerializerTest extends SerializerTestBase { +public class ShortValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializerTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializerTest.java index 52892caf15c97..7970025164620 100644 --- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializerTest.java +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializerTest.java @@ -18,7 +18,6 @@ package org.apache.flink.graph.types.valuearray; -import org.apache.flink.api.common.typeutils.SerializerTestBase; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.types.StringValue; @@ -27,7 +26,7 @@ /** * A test for the {@link StringValueArraySerializer}. */ -public class StringValueArraySerializerTest extends SerializerTestBase { +public class StringValueArraySerializerTest extends ValueArraySerializerTestBase { @Override protected TypeSerializer createSerializer() { diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ValueArraySerializerTestBase.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ValueArraySerializerTestBase.java new file mode 100644 index 0000000000000..81aa3a11c8dd3 --- /dev/null +++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/types/valuearray/ValueArraySerializerTestBase.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.graph.types.valuearray; + +import org.apache.flink.api.common.typeutils.SerializerTestBase; +import org.apache.flink.testutils.DeeplyEqualsChecker; + +import java.util.Objects; + +/** + * Base class for tests of {@link org.apache.flink.api.common.typeutils.TypeSerializer}s for any {@link ValueArray}. + * It overrides default deepEquals of {@link Iterable}s with {@link Objects#equals(Object, Object)}. + */ +public abstract class ValueArraySerializerTestBase> extends SerializerTestBase { + ValueArraySerializerTestBase() { + super( + new DeeplyEqualsChecker() + .withCustomCheck( + (o1, o2) -> o1 instanceof ValueArray && o2 instanceof ValueArray, + (o1, o2, checker) -> Objects.equals(o1, o2))); + } + +} diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java index d25f3d51ffe71..3e6a068d8aafb 100644 --- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java +++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java @@ -126,7 +126,9 @@ public void close() { process.destroy(); } finally { try { - server.close(); + if (server != null) { + server.close(); + } } catch (IOException e) { LOG.error("Failed to close socket.", e); } diff --git a/flink-libraries/flink-sql-client/pom.xml b/flink-libraries/flink-sql-client/pom.xml index c0299b8f31ebb..823c8366cb31c 100644 --- a/flink-libraries/flink-sql-client/pom.xml +++ b/flink-libraries/flink-sql-client/pom.xml @@ -144,30 +144,19 @@ under the License. + shade-flink package shade - - - *:* - - - org/apache/calcite/** - org/apache/flink/calcite/shaded/** - org/apache/flink/table/** - org/apache/flink/cep/** - org.codehaus.commons.compiler.properties - org/codehaus/janino/** - org/codehaus/commons/** - META-INF/services/org.apache.flink.table.factories.TableFactory - - org/jline/** - com/fasterxml/jackson/** - - - + + + org.apache.flink:flink-table_${scala.binary.version} + org.apache.flink:flink-cep_${scala.binary.version} + org.jline:* + + diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClient.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClient.java index 757099b4d7140..0a7a0afc082eb 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClient.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClient.java @@ -115,16 +115,23 @@ private void start() { * @param executor executor */ private void openCli(SessionContext context, Executor executor) { - final CliClient cli = new CliClient(context, executor); - // interactive CLI mode - if (options.getUpdateStatement() == null) { - cli.open(); - } - // execute single update statement - else { - final boolean success = cli.submitUpdate(options.getUpdateStatement()); - if (!success) { - throw new SqlClientException("Could not submit given SQL update statement to cluster."); + CliClient cli = null; + try { + cli = new CliClient(context, executor); + // interactive CLI mode + if (options.getUpdateStatement() == null) { + cli.open(); + } + // execute single update statement + else { + final boolean success = cli.submitUpdate(options.getUpdateStatement()); + if (!success) { + throw new SqlClientException("Could not submit given SQL update statement to cluster."); + } + } + } finally { + if (cli != null) { + cli.close(); } } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java index 5388a3e89f09f..71bfe16689baf 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java @@ -119,7 +119,7 @@ protected void refresh() { break; // stop retrieval if job is done case EOS: - stopRetrieval(); + stopRetrieval(false); break; default: List> changes = result.getPayload(); diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliClient.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliClient.java index 987032efd27aa..a7c98938460c2 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliClient.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliClient.java @@ -18,6 +18,7 @@ package org.apache.flink.table.client.cli; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.client.SqlClientException; import org.apache.flink.table.client.cli.SqlCommandParser.SqlCommandCall; @@ -77,21 +78,19 @@ public class CliClient { private static final int SOURCE_MAX_SIZE = 50_000; - public CliClient(SessionContext context, Executor executor) { + /** + * Creates a CLI instance with a custom terminal. Make sure to close the CLI instance + * afterwards using {@link #close()}. + */ + @VisibleForTesting + public CliClient(Terminal terminal, SessionContext context, Executor executor) { + this.terminal = terminal; this.context = context; this.executor = executor; - try { - // initialize terminal - terminal = TerminalBuilder.builder() - .name(CliStrings.CLI_NAME) - .build(); - // make space from previous output and test the writer - terminal.writer().println(); - terminal.writer().flush(); - } catch (IOException e) { - throw new SqlClientException("Error opening command line interface.", e); - } + // make space from previous output and test the writer + terminal.writer().println(); + terminal.writer().flush(); // initialize line lineReader lineReader = LineReaderBuilder.builder() @@ -117,6 +116,14 @@ public CliClient(SessionContext context, Executor executor) { .toAnsi(); } + /** + * Creates a CLI instance with a prepared terminal. Make sure to close the CLI instance + * afterwards using {@link #close()}. + */ + public CliClient(SessionContext context, Executor executor) { + this(createDefaultTerminal(), context, executor); + } + public Terminal getTerminal() { return terminal; } @@ -194,6 +201,17 @@ public void open() { } } + /** + * Closes the CLI instance. + */ + public void close() { + try { + terminal.close(); + } catch (IOException e) { + throw new SqlClientException("Unable to close terminal.", e); + } + } + /** * Submits a SQL update statement and prints status information and/or errors on the terminal. * @@ -530,4 +548,16 @@ private void printInfo(String message) { terminal.writer().println(CliStrings.messageInfo(message).toAnsi()); terminal.flush(); } + + // -------------------------------------------------------------------------------------------- + + private static Terminal createDefaultTerminal() { + try { + return TerminalBuilder.builder() + .name(CliStrings.CLI_NAME) + .build(); + } catch (IOException e) { + throw new SqlClientException("Error opening command line interface.", e); + } + } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java index 1fdd005cd6375..67ce5ebb21141 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java @@ -150,8 +150,9 @@ protected void openRow() { view.open(); // enter view } - protected void stopRetrieval() { + protected void stopRetrieval(boolean cleanUpQuery) { // stop retrieval + refreshThread.cleanUpQuery = cleanUpQuery; refreshThread.isRunning = false; synchronized (refreshThread) { refreshThread.notify(); @@ -217,7 +218,7 @@ protected List computeMainLines() { @Override protected void cleanUp() { - stopRetrieval(); + stopRetrieval(true); } // -------------------------------------------------------------------------------------------- @@ -226,6 +227,8 @@ private class RefreshThread extends Thread { public volatile boolean isRunning = true; + public volatile boolean cleanUpQuery = true; + public long lastUpdatedResults = System.currentTimeMillis(); @Override @@ -278,13 +281,15 @@ public void run() { } } - // cancel table program - try { - // the cancellation happens in the refresh thread in order to keep the main thread - // responsive at all times; esp. if the cluster is not available - client.getExecutor().cancelQuery(client.getContext(), resultDescriptor.getResultId()); - } catch (SqlExecutionException e) { - // ignore further exceptions + if (cleanUpQuery) { + // cancel table program + try { + // the cancellation happens in the refresh thread in order to keep the main thread + // responsive at all times; esp. if the cluster is not available + client.getExecutor().cancelQuery(client.getContext(), resultDescriptor.getResultId()); + } catch (SqlExecutionException e) { + // ignore further exceptions + } } } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java index 9c218f6d89ead..9a8cbcc9a8e7c 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java @@ -96,7 +96,7 @@ protected void refresh() { // stop retrieval if job is done if (result.getType() == TypedResult.ResultType.EOS) { - stopRetrieval(); + stopRetrieval(false); } // update page else if (result.getType() == TypedResult.ResultType.PAYLOAD) { diff --git a/flink-libraries/flink-sql-client/src/main/resources/META-INF/NOTICE b/flink-libraries/flink-sql-client/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..9b039bbe1b1f4 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/main/resources/META-INF/NOTICE @@ -0,0 +1,11 @@ +flink-sql-client +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- org.jline:jline-terminal:3.9.0 +- org.jline:jline-reader:3.9.0 diff --git a/flink-libraries/flink-sql-client/src/main/resources/META-INF/licenses/LICENSE.jline b/flink-libraries/flink-sql-client/src/main/resources/META-INF/licenses/LICENSE.jline new file mode 100644 index 0000000000000..e34763968c262 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/main/resources/META-INF/licenses/LICENSE.jline @@ -0,0 +1,7 @@ +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java index 4bf284b3ccb0e..2f8f2b59cc0e3 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java @@ -20,6 +20,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.client.cli.utils.TerminalUtils; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.client.gateway.Executor; import org.apache.flink.table.client.gateway.ProgramTargetDescriptor; @@ -35,8 +36,10 @@ import org.jline.reader.LineReaderBuilder; import org.jline.reader.ParsedLine; import org.jline.reader.Parser; +import org.jline.terminal.Terminal; import org.junit.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -70,7 +73,7 @@ public void testFailedUpdateSubmission() { } @Test - public void testSqlCompletion() { + public void testSqlCompletion() throws IOException { verifySqlCompletion("", 0, Arrays.asList("SELECT", "QUIT;", "RESET;"), Collections.emptyList()); verifySqlCompletion("SELEC", 5, Collections.singletonList("SELECT"), Collections.singletonList("QUIT;")); verifySqlCompletion("SELE", 0, Collections.singletonList("SELECT"), Collections.singletonList("QUIT;")); @@ -88,41 +91,51 @@ private void verifyUpdateSubmission(String statement, boolean failExecution, boo final MockExecutor mockExecutor = new MockExecutor(); mockExecutor.failExecution = failExecution; - final CliClient client = new CliClient(context, mockExecutor); - if (testFailure) { - assertFalse(client.submitUpdate(statement)); - } else { - assertTrue(client.submitUpdate(statement)); - assertEquals(statement, mockExecutor.receivedStatement); - assertEquals(context, mockExecutor.receivedContext); + CliClient cli = null; + try { + cli = new CliClient(TerminalUtils.createDummyTerminal(), context, mockExecutor); + if (testFailure) { + assertFalse(cli.submitUpdate(statement)); + } else { + assertTrue(cli.submitUpdate(statement)); + assertEquals(statement, mockExecutor.receivedStatement); + assertEquals(context, mockExecutor.receivedContext); + } + } finally { + if (cli != null) { + cli.close(); + } } } - private void verifySqlCompletion(String statement, int position, List expectedHints, List notExpectedHints) { + private void verifySqlCompletion(String statement, int position, List expectedHints, List notExpectedHints) throws IOException { final SessionContext context = new SessionContext("test-session", new Environment()); final MockExecutor mockExecutor = new MockExecutor(); final SqlCompleter completer = new SqlCompleter(context, mockExecutor); final SqlMultiLineParser parser = new SqlMultiLineParser(); - final LineReader reader = LineReaderBuilder.builder().build(); - final ParsedLine parsedLine = parser.parse(statement, position, Parser.ParseContext.COMPLETE); - final List candidates = new ArrayList<>(); - final List results = new ArrayList<>(); - completer.complete(reader, parsedLine, candidates); - candidates.forEach(item -> results.add(item.value())); + try (Terminal terminal = TerminalUtils.createDummyTerminal()) { + final LineReader reader = LineReaderBuilder.builder().terminal(terminal).build(); - assertTrue(results.containsAll(expectedHints)); + final ParsedLine parsedLine = parser.parse(statement, position, Parser.ParseContext.COMPLETE); + final List candidates = new ArrayList<>(); + final List results = new ArrayList<>(); + completer.complete(reader, parsedLine, candidates); + candidates.forEach(item -> results.add(item.value())); - assertEquals(statement, mockExecutor.receivedStatement); - assertEquals(context, mockExecutor.receivedContext); - assertEquals(position, mockExecutor.receivedPosition); - assertTrue(results.contains("HintA")); - assertTrue(results.contains("Hint B")); + assertTrue(results.containsAll(expectedHints)); - results.retainAll(notExpectedHints); - assertEquals(0, results.size()); + assertEquals(statement, mockExecutor.receivedStatement); + assertEquals(context, mockExecutor.receivedContext); + assertEquals(position, mockExecutor.receivedPosition); + assertTrue(results.contains("HintA")); + assertTrue(results.contains("Hint B")); + + results.retainAll(notExpectedHints); + assertEquals(0, results.size()); + } } // -------------------------------------------------------------------------------------------- diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliResultViewTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliResultViewTest.java new file mode 100644 index 0000000000000..0d090bd3f8766 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliResultViewTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.cli; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.Types; +import org.apache.flink.table.client.cli.utils.TerminalUtils; +import org.apache.flink.table.client.config.Environment; +import org.apache.flink.table.client.gateway.Executor; +import org.apache.flink.table.client.gateway.ProgramTargetDescriptor; +import org.apache.flink.table.client.gateway.ResultDescriptor; +import org.apache.flink.table.client.gateway.SessionContext; +import org.apache.flink.table.client.gateway.SqlExecutionException; +import org.apache.flink.table.client.gateway.TypedResult; +import org.apache.flink.types.Row; + +import org.jline.utils.AttributedString; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +/** + * Contains basic tests for the {@link CliResultView}. + */ +public class CliResultViewTest { + + @Test + public void testTableResultViewKeepJobResult() throws Exception { + testResultViewClearResult(TypedResult.endOfStream(), true, 0); + } + + @Test + public void testTableResultViewClearEmptyResult() throws Exception { + testResultViewClearResult(TypedResult.empty(), true, 1); + } + + @Test + public void testTableResultViewClearPayloadResult() throws Exception { + testResultViewClearResult(TypedResult.payload(1), true, 1); + } + + @Test + public void testChangelogResultViewKeepJobResult() throws Exception { + testResultViewClearResult(TypedResult.endOfStream(), false, 0); + } + + @Test + public void testChangelogResultViewClearEmptyResult() throws Exception { + testResultViewClearResult(TypedResult.empty(), false, 1); + } + + @Test + public void testChangelogResultViewClearPayloadResult() throws Exception { + testResultViewClearResult(TypedResult.payload(Collections.emptyList()), false, 1); + } + + private void testResultViewClearResult(TypedResult typedResult, boolean isTableMode, int expectedCancellationCount) throws Exception { + final CountDownLatch cancellationCounterLatch = new CountDownLatch(expectedCancellationCount); + final SessionContext session = new SessionContext("test-session", new Environment()); + final MockExecutor executor = new MockExecutor(typedResult, cancellationCounterLatch); + final ResultDescriptor descriptor = new ResultDescriptor( + "result-id", + TableSchema.builder().field("Null Field", Types.STRING()).build(), + false); + + Thread resultViewRunner = null; + CliClient cli = null; + try { + cli = new CliClient(TerminalUtils.createDummyTerminal(), session, executor); + resultViewRunner = new Thread(new TestingCliResultView(cli, descriptor, isTableMode)); + resultViewRunner.start(); + } finally { + if (resultViewRunner != null && !resultViewRunner.isInterrupted()) { + resultViewRunner.interrupt(); + } + if (cli != null) { + cli.close(); + } + } + + assertTrue( + "Invalid number of cancellations.", + cancellationCounterLatch.await(10, TimeUnit.SECONDS)); + } + + private static final class MockExecutor implements Executor { + + private final TypedResult typedResult; + private final CountDownLatch cancellationCounter; + + public MockExecutor(TypedResult typedResult, CountDownLatch cancellationCounter) { + this.typedResult = typedResult; + this.cancellationCounter = cancellationCounter; + } + + @Override + public void start() throws SqlExecutionException { + // do nothing + } + + @Override + public Map getSessionProperties(SessionContext session) throws SqlExecutionException { + return null; + } + + @Override + public List listTables(SessionContext session) throws SqlExecutionException { + return null; + } + + @Override + public List listUserDefinedFunctions(SessionContext session) throws SqlExecutionException { + return null; + } + + @Override + public TableSchema getTableSchema(SessionContext session, String name) throws SqlExecutionException { + return null; + } + + @Override + public String explainStatement(SessionContext session, String statement) throws SqlExecutionException { + return null; + } + + @Override + public List completeStatement(SessionContext session, String statement, int position) { + return null; + } + + @Override + public ResultDescriptor executeQuery(SessionContext session, String query) throws SqlExecutionException { + return null; + } + + @Override + @SuppressWarnings("unchecked") + public TypedResult>> retrieveResultChanges(SessionContext session, String resultId) throws SqlExecutionException { + return (TypedResult>>) typedResult; + } + + @Override + @SuppressWarnings("unchecked") + public TypedResult snapshotResult(SessionContext session, String resultId, int pageSize) throws SqlExecutionException { + return (TypedResult) typedResult; + } + + @Override + public List retrieveResultPage(String resultId, int page) throws SqlExecutionException { + return Collections.singletonList(new Row(1)); + } + + @Override + public void cancelQuery(SessionContext session, String resultId) throws SqlExecutionException { + cancellationCounter.countDown(); + } + + @Override + public ProgramTargetDescriptor executeUpdate(SessionContext session, String statement) throws SqlExecutionException { + return null; + } + + @Override + public void validateSession(SessionContext session) throws SqlExecutionException { + // do nothing + } + + @Override + public void stop(SessionContext session) { + // do nothing + } + } + + private static final class TestingCliResultView implements Runnable { + + private final CliResultView realResultView; + + public TestingCliResultView( + CliClient client, + ResultDescriptor descriptor, + boolean isTableMode) { + + if (isTableMode) { + realResultView = new TestingCliTableResultView(client, descriptor); + } else { + realResultView = new TestingCliChangelogResultView(client, descriptor); + } + } + + @Override + public void run() { + realResultView.open(); + } + } + + private static class TestingCliChangelogResultView extends CliChangelogResultView { + + public TestingCliChangelogResultView(CliClient client, ResultDescriptor resultDescriptor) { + super(client, resultDescriptor); + } + + @Override + protected List computeMainHeaderLines() { + return Collections.emptyList(); + } + } + + private static class TestingCliTableResultView extends CliTableResultView { + + public TestingCliTableResultView(CliClient client, ResultDescriptor resultDescriptor) { + super(client, resultDescriptor); + } + + @Override + protected List computeMainHeaderLines() { + return Collections.emptyList(); + } + } +} diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/TerminalUtils.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/TerminalUtils.java new file mode 100644 index 0000000000000..772fb013e3c3c --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/TerminalUtils.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.cli.utils; + +import org.jline.terminal.Terminal; +import org.jline.terminal.impl.DumbTerminal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Utilities for terminal handling. + */ +public class TerminalUtils { + + private TerminalUtils() { + // do not instantiate + } + + public static Terminal createDummyTerminal() { + try { + return new DumbTerminal(new MockInputStream(), new MockOutputStream()); + } catch (IOException e) { + throw new RuntimeException("Unable to create dummy terminal."); + } + } + + private static class MockInputStream extends InputStream { + + @Override + public int read() { + return 0; + } + } + + private static class MockOutputStream extends OutputStream { + + @Override + public void write(int b) { + // do nothing + } + } +} diff --git a/flink-libraries/flink-streaming-python/pom.xml b/flink-libraries/flink-streaming-python/pom.xml index 212fcc09ff9ed..b25fa6d28967c 100644 --- a/flink-libraries/flink-streaming-python/pom.xml +++ b/flink-libraries/flink-streaming-python/pom.xml @@ -61,7 +61,7 @@ under the License. org.python jython-standalone - 2.7.0 + 2.7.1 diff --git a/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..6fcb19f3bc146 --- /dev/null +++ b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE @@ -0,0 +1,10 @@ +flink-streaming-python +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php) +See bundled license files for details. + +- org.python:jython-standalone:2.7.1 diff --git a/flink-libraries/flink-streaming-python/src/main/resources/META-INF/licenses/LICENSE.jython b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/licenses/LICENSE.jython new file mode 100644 index 0000000000000..1c2f7ffeb5bc4 --- /dev/null +++ b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/licenses/LICENSE.jython @@ -0,0 +1,17 @@ +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 + +1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Jython") in source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Jython alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2007 Python Software Foundation; All Rights Reserved" are retained in Jython alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on or incorporates Jython or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Jython. + +4. PSF is making Jython available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF JYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF JYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING JYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Jython, Licensee agrees to be bound by the terms and conditions of this License Agreement. diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java index a1d3e783cdc1c..ed62b46b36b50 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java @@ -68,7 +68,7 @@ protected final boolean isFormatNeeded() { /** * Converts this descriptor into a set of connector properties. Usually prefixed with - * {@link FormatDescriptorValidator#FORMAT}. + * {@link ConnectorDescriptorValidator#CONNECTOR}. */ protected abstract Map toConnectorProperties(); } diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java index 88cf34905f452..cf9860ae69e2a 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java @@ -26,6 +26,11 @@ @Internal public abstract class ConnectorDescriptorValidator implements DescriptorValidator { + /** + * Prefix for connector-related properties. + */ + public static final String CONNECTOR = "connector"; + /** * Key for describing the type of the connector. Usually used for factory discovery. */ diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java index c47698147284e..76289128b5bca 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java @@ -938,7 +938,7 @@ public void validateFixedIndexedProperties(String key, boolean allowEmpty, Map> subKey : subKeyValidation.entrySet()) { final String fullKey = key + '.' + i + '.' + subKey.getKey(); if (properties.containsKey(fullKey)) { @@ -1134,7 +1134,7 @@ public void validateArray(String key, Consumer elementValidation, int mi } // validate array elements - for (int i = 0; i < maxIndex; i++) { + for (int i = 0; i <= maxIndex; i++) { final String fullKey = key + '.' + i; if (properties.containsKey(fullKey)) { // run validation logic diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java index 47aac25e89746..5531082611d36 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java @@ -76,7 +76,7 @@ public static T decodeStringToObject(String base64Strin return instance; } catch (Exception e) { throw new ValidationException( - "Unable to deserialize string '" + base64String + "' of base class '" + baseClass.getName() + "'."); + "Unable to deserialize string '" + base64String + "' of base class '" + baseClass.getName() + "'.", e); } } diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java new file mode 100644 index 0000000000000..cca84d28d17db --- /dev/null +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataview; + +import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.api.dataview.ListView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.util.List; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A {@link TypeSerializerSnapshot} for the {@link ListViewSerializer}. + * + * @param the type of the list elements. + */ +public final class ListViewSerializerSnapshot implements TypeSerializerSnapshot> { + + private static final int CURRENT_VERSION = 1; + + private CompositeSerializerSnapshot nestedListSerializerSnapshot; + + /** + * Constructor for read instantiation. + */ + public ListViewSerializerSnapshot() {} + + /** + * Constructor to create the snapshot for writing. + */ + public ListViewSerializerSnapshot(TypeSerializer> listSerializer) { + this.nestedListSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(listSerializer)); + } + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public TypeSerializer> restoreSerializer() { + return new ListViewSerializer<>(nestedListSerializerSnapshot.getRestoreSerializer(0)); + } + + @Override + public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { + checkState(nestedListSerializerSnapshot != null); + + if (newSerializer instanceof ListViewSerializer) { + ListViewSerializer serializer = (ListViewSerializer) newSerializer; + + return nestedListSerializerSnapshot.resolveCompatibilityWithNested( + TypeSerializerSchemaCompatibility.compatibleAsIs(), + serializer.getListSerializer()); + } + else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + nestedListSerializerSnapshot.writeCompositeSnapshot(out); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.nestedListSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + } +} diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java deleted file mode 100644 index 10b4419a8c7c9..0000000000000 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.dataview; - -import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.base.MapSerializer; -import org.apache.flink.table.api.dataview.MapView; - -/** - * A {@link TypeSerializerConfigSnapshot} for the {@link MapViewSerializer}. - * - * @param the key type of the map entries. - * @param the value type of the map entries. - */ -public class MapViewSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - - private static final int VERSION = 1; - - /** This empty nullary constructor is required for deserializing the configuration. */ - public MapViewSerializerConfigSnapshot() {} - - public MapViewSerializerConfigSnapshot(MapSerializer mapSerializer) { - super(mapSerializer); - } - - @Override - public int getVersion() { - return VERSION; - } -} diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java new file mode 100644 index 0000000000000..f59fc0a365423 --- /dev/null +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataview; + +import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.api.dataview.MapView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A {@link TypeSerializerSnapshot} for the {@link MapViewSerializer}. + * + * @param the key type of the map entries. + * @param the value type of the map entries. + */ +public class MapViewSerializerSnapshot implements TypeSerializerSnapshot> { + + private static final int CURRENT_VERSION = 1; + + private CompositeSerializerSnapshot nestedMapSerializerSnapshot; + + /** + * Constructor for read instantiation. + */ + public MapViewSerializerSnapshot() {} + + /** + * Constructor to create the snapshot for writing. + */ + public MapViewSerializerSnapshot(TypeSerializer> mapSerializer) { + this.nestedMapSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(mapSerializer)); + } + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public TypeSerializer> restoreSerializer() { + return new MapViewSerializer<>(nestedMapSerializerSnapshot.getRestoreSerializer(0)); + } + + @Override + public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility( + TypeSerializer> newSerializer) { + checkState(nestedMapSerializerSnapshot != null); + + if (newSerializer instanceof MapViewSerializer) { + MapViewSerializer serializer = (MapViewSerializer) newSerializer; + + return nestedMapSerializerSnapshot.resolveCompatibilityWithNested( + TypeSerializerSchemaCompatibility.compatibleAsIs(), + serializer.getMapSerializer()); + } + else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + nestedMapSerializerSnapshot.writeCompositeSnapshot(out); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.nestedMapSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + } +} diff --git a/flink-libraries/flink-table/src/main/resources/META-INF/NOTICE b/flink-libraries/flink-table/src/main/resources/META-INF/NOTICE index bfe2737d6d110..54a253575ac5a 100644 --- a/flink-libraries/flink-table/src/main/resources/META-INF/NOTICE +++ b/flink-libraries/flink-table/src/main/resources/META-INF/NOTICE @@ -1,64 +1,21 @@ -=========================================================== - NOTICE corresponding to the section 4d of - The Apache License, Version 2.0, for - flink-table -=========================================================== +flink-table +Copyright 2014-2018 The Apache Software Foundation -The 'flink-table' module bundles certain dependencies into -the binary release artifacts. -Below are the license statements where required by the license. +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -Janino version (org.codehaus.janino : janino) v. 3.0.7 -under the following license: - - Janino - An embedded Java[TM] compiler - - Copyright (c) 2001-2016, Arno Unkrig - Copyright (c) 2015-2016 TIBCO Software Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - 3. Neither the name of JANINO nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN - IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) +- com.esri.geometry:esri-geometry-api:2.0.0 +- com.google.guava:guava:19.0 +- joda-time:joda-time:2.5 +- net.hydromatic:aggdesigner-algorithm:6.0 +- org.apache.calcite:calcite-core:1.17.0 +- org.apache.calcite:calcite-linq4j:1.17.0 +- org.apache.calcite.avatica:avatica-core:1.12.0 -Reflections (org.reflections : reflections) v. 0.9.10 -under the following license: +This project bundles the following dependencies under the BSD license. +See bundled license files for details - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - - \ No newline at end of file +- org.codehaus.janino:janino:3.0.7 +- org.codehaus.janino:commons-compiler:3.0.7 diff --git a/flink-libraries/flink-table/src/main/resources/META-INF/licenses/LICENSE.janino b/flink-libraries/flink-table/src/main/resources/META-INF/licenses/LICENSE.janino new file mode 100644 index 0000000000000..ef871e2426218 --- /dev/null +++ b/flink-libraries/flink-table/src/main/resources/META-INF/licenses/LICENSE.janino @@ -0,0 +1,31 @@ +Janino - An embedded Java[TM] compiler + +Copyright (c) 2001-2016, Arno Unkrig +Copyright (c) 2015-2016 TIBCO Software Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of JANINO nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala index 6a7a921bffbe3..99e9d7e6d011b 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala @@ -449,9 +449,8 @@ abstract class BatchTableEnvironment( */ private[flink] def optimize(relNode: RelNode): RelNode = { val convSubQueryPlan = optimizeConvertSubQueries(relNode) - val temporalTableJoinPlan = optimizeConvertToTemporalJoin(convSubQueryPlan) - val fullNode = optimizeConvertTableReferences(temporalTableJoinPlan) - val decorPlan = RelDecorrelator.decorrelateQuery(fullNode) + val expandedPlan = optimizeExpandPlan(convSubQueryPlan) + val decorPlan = RelDecorrelator.decorrelateQuery(expandedPlan) val normalizedPlan = optimizeNormalizeLogicalPlan(decorPlan) val logicalPlan = optimizeLogicalPlan(normalizedPlan) optimizePhysicalPlan(logicalPlan, FlinkConventions.DATASET) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala index 4973f34147f61..8c6a1e0a04bf4 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala @@ -804,13 +804,13 @@ abstract class StreamTableEnvironment( */ private[flink] def optimize(relNode: RelNode, updatesAsRetraction: Boolean): RelNode = { val convSubQueryPlan = optimizeConvertSubQueries(relNode) - val temporalTableJoinPlan = optimizeConvertToTemporalJoin(convSubQueryPlan) - val fullNode = optimizeConvertTableReferences(temporalTableJoinPlan) - val decorPlan = RelDecorrelator.decorrelateQuery(fullNode) + val expandedPlan = optimizeExpandPlan(convSubQueryPlan) + val decorPlan = RelDecorrelator.decorrelateQuery(expandedPlan) val planWithMaterializedTimeAttributes = RelTimeIndicatorConverter.convert(decorPlan, getRelBuilder.getRexBuilder) val normalizedPlan = optimizeNormalizeLogicalPlan(planWithMaterializedTimeAttributes) val logicalPlan = optimizeLogicalPlan(normalizedPlan) + val physicalPlan = optimizePhysicalPlan(logicalPlan, FlinkConventions.DATASTREAM) optimizeDecoratePlan(physicalPlan, updatesAsRetraction) } @@ -827,7 +827,7 @@ abstract class StreamTableEnvironment( } else { relNode } - runHepPlanner( + runHepPlannerSequentially( HepMatchOrder.BOTTOM_UP, decoRuleSet, planToDecorate, diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala index 58831d1027122..e28a471681d2c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList import org.apache.calcite.config.Lex import org.apache.calcite.jdbc.CalciteSchema import org.apache.calcite.plan.RelOptPlanner.CannotPlanException -import org.apache.calcite.plan.hep.{HepMatchOrder, HepPlanner, HepProgramBuilder} +import org.apache.calcite.plan.hep.{HepMatchOrder, HepPlanner, HepProgram, HepProgramBuilder} import org.apache.calcite.plan.{Convention, RelOptPlanner, RelOptUtil, RelTraitSet} import org.apache.calcite.rel.RelNode import org.apache.calcite.schema.SchemaPlus @@ -91,6 +91,7 @@ abstract class TableEnvironment(val config: TableConfig) { .costFactory(new DataSetCostFactory) .typeSystem(new FlinkTypeSystem) .operatorTable(getSqlOperatorTable) + .sqlToRelConverterConfig(getSqlToRelConverterConfig) // set the executor to evaluate constant expressions .executor(new ExpressionReducer(config)) .build @@ -109,15 +110,6 @@ abstract class TableEnvironment(val config: TableConfig) { // registered external catalog names -> catalog private val externalCatalogs = new mutable.HashMap[String, ExternalCatalog] - // configuration for SqlToRelConverter - private[flink] lazy val sqlToRelConverterConfig: SqlToRelConverter.Config = { - val calciteConfig = config.getCalciteConfig - calciteConfig.getSqlToRelConverterConfig match { - case Some(c) => c - case None => getSqlToRelConverterConfig - } - } - /** Returns the table config to define the runtime behavior of the Table API. */ def getConfig: TableConfig = config @@ -132,11 +124,18 @@ abstract class TableEnvironment(val config: TableConfig) { * Returns the SqlToRelConverter config. */ protected def getSqlToRelConverterConfig: SqlToRelConverter.Config = { - SqlToRelConverter.configBuilder() - .withTrimUnusedFields(false) - .withConvertTableAccess(false) - .withInSubQueryThreshold(Integer.MAX_VALUE) - .build() + val calciteConfig = config.getCalciteConfig + calciteConfig.getSqlToRelConverterConfig match { + + case None => + SqlToRelConverter.configBuilder() + .withTrimUnusedFields(false) + .withConvertTableAccess(false) + .withInSubQueryThreshold(Integer.MAX_VALUE) + .build() + + case Some(c) => c + } } /** @@ -256,34 +255,31 @@ abstract class TableEnvironment(val config: TableConfig) { protected def getBuiltInPhysicalOptRuleSet: RuleSet protected def optimizeConvertSubQueries(relNode: RelNode): RelNode = { - runHepPlanner( + runHepPlannerSequentially( HepMatchOrder.BOTTOM_UP, FlinkRuleSets.TABLE_SUBQUERY_RULES, relNode, relNode.getTraitSet) } - protected def optimizeConvertToTemporalJoin(relNode: RelNode): RelNode = { - runHepPlanner( - HepMatchOrder.BOTTOM_UP, - FlinkRuleSets.TEMPORAL_JOIN_RULES, + protected def optimizeExpandPlan(relNode: RelNode): RelNode = { + val result = runHepPlannerSimultaneously( + HepMatchOrder.TOP_DOWN, + FlinkRuleSets.EXPAND_PLAN_RULES, relNode, relNode.getTraitSet) - } - protected def optimizeConvertTableReferences(relNode: RelNode): RelNode = { - runHepPlanner( - HepMatchOrder.BOTTOM_UP, - FlinkRuleSets.TABLE_REF_RULES, - relNode, - relNode.getTraitSet) + runHepPlannerSequentially( + HepMatchOrder.TOP_DOWN, + FlinkRuleSets.POST_EXPAND_CLEAN_UP_RULES, + result, + result.getTraitSet) } - protected def optimizeNormalizeLogicalPlan(relNode: RelNode): RelNode = { val normRuleSet = getNormRuleSet if (normRuleSet.iterator().hasNext) { - runHepPlanner(HepMatchOrder.BOTTOM_UP, normRuleSet, relNode, relNode.getTraitSet) + runHepPlannerSequentially(HepMatchOrder.BOTTOM_UP, normRuleSet, relNode, relNode.getTraitSet) } else { relNode } @@ -310,13 +306,16 @@ abstract class TableEnvironment(val config: TableConfig) { } /** - * run HEP planner + * run HEP planner with rules applied one by one. First apply one rule to all of the nodes + * and only then apply the next rule. If a rule creates a new node preceding rules will not + * be applied to the newly created node. */ - protected def runHepPlanner( + protected def runHepPlannerSequentially( hepMatchOrder: HepMatchOrder, ruleSet: RuleSet, input: RelNode, targetTraits: RelTraitSet): RelNode = { + val builder = new HepProgramBuilder builder.addMatchOrder(hepMatchOrder) @@ -324,8 +323,36 @@ abstract class TableEnvironment(val config: TableConfig) { while (it.hasNext) { builder.addRuleInstance(it.next()) } + runHepPlanner(builder.build(), input, targetTraits) + } + + /** + * run HEP planner with rules applied simultaneously. Apply all of the rules to the given + * node before going to the next one. If a rule creates a new node all of the rules will + * be applied to this new node. + */ + protected def runHepPlannerSimultaneously( + hepMatchOrder: HepMatchOrder, + ruleSet: RuleSet, + input: RelNode, + targetTraits: RelTraitSet): RelNode = { + + val builder = new HepProgramBuilder + builder.addMatchOrder(hepMatchOrder) - val planner = new HepPlanner(builder.build, frameworkConfig.getContext) + builder.addRuleCollection(ruleSet.asScala.toList.asJava) + runHepPlanner(builder.build(), input, targetTraits) + } + + /** + * run HEP planner + */ + protected def runHepPlanner( + hepProgram: HepProgram, + input: RelNode, + targetTraits: RelTraitSet): RelNode = { + + val planner = new HepPlanner(hepProgram, frameworkConfig.getContext) planner.setRoot(input) if (input.getTraitSet != targetTraits) { planner.changeTraits(input, targetTraits.simplify) @@ -689,8 +716,7 @@ abstract class TableEnvironment(val config: TableConfig) { val planner = new FlinkPlannerImpl( getFrameworkConfig, getPlanner, - getTypeFactory, - sqlToRelConverterConfig) + getTypeFactory) planner.getCompletionHints(statement, position) } @@ -712,8 +738,7 @@ abstract class TableEnvironment(val config: TableConfig) { * @return The result of the query as Table */ def sqlQuery(query: String): Table = { - val planner = new FlinkPlannerImpl( - getFrameworkConfig, getPlanner, getTypeFactory, sqlToRelConverterConfig) + val planner = new FlinkPlannerImpl(getFrameworkConfig, getPlanner, getTypeFactory) // parse the sql query val parsed = planner.parse(query) if (null != parsed && parsed.getKind.belongsTo(SqlKind.QUERY)) { @@ -773,8 +798,7 @@ abstract class TableEnvironment(val config: TableConfig) { * @param config The [[QueryConfig]] to use. */ def sqlUpdate(stmt: String, config: QueryConfig): Unit = { - val planner = new FlinkPlannerImpl( - getFrameworkConfig, getPlanner, getTypeFactory, sqlToRelConverterConfig) + val planner = new FlinkPlannerImpl(getFrameworkConfig, getPlanner, getTypeFactory) // parse the sql query val parsed = planner.parse(stmt) parsed match { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala index ca300539f7b33..400279d8c7268 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala @@ -48,8 +48,7 @@ import scala.collection.JavaConversions._ class FlinkPlannerImpl( config: FrameworkConfig, planner: RelOptPlanner, - typeFactory: FlinkTypeFactory, - sqlToRelConverterConfig: SqlToRelConverter.Config) { + typeFactory: FlinkTypeFactory) { val operatorTable: SqlOperatorTable = config.getOperatorTable /** Holds the trait definitions to be registered with planner. May be null. */ @@ -57,6 +56,7 @@ class FlinkPlannerImpl( val parserConfig: SqlParser.Config = config.getParserConfig val convertletTable: SqlRexConvertletTable = config.getConvertletTable val defaultSchema: SchemaPlus = config.getDefaultSchema + val sqlToRelConverterConfig: SqlToRelConverter.Config = config.getSqlToRelConverterConfig var validator: FlinkCalciteSqlValidator = _ var root: RelRoot = _ diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala index 41f0fc5f685c4..c1bcf1438d354 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala @@ -23,6 +23,7 @@ import org.apache.calcite.rel.core._ import org.apache.calcite.rel.logical._ import org.apache.calcite.rel.{RelNode, RelShuttle} import org.apache.calcite.rex._ +import org.apache.calcite.sql.`type`.SqlTypeName import org.apache.calcite.sql.fun.SqlStdOperatorTable import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo import org.apache.flink.table.api.{TableException, ValidationException} @@ -100,13 +101,7 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { override def visit(matchRel: LogicalMatch): RelNode = { // visit children and update inputs val input = matchRel.getInput.accept(this) - - // check if input field contains time indicator type - // materialize field if no time indicator is present anymore - // if input field is already materialized, change to timestamp type - val materializer = new RexTimeIndicatorMaterializer( - rexBuilder, - input.getRowType.getFieldList.map(_.getType)) + val materializer = createMaterializer(input) // update input expressions val patternDefs = matchRel.getPatternDefinitions.mapValues(_.accept(materializer)) @@ -180,23 +175,16 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { override def visit(filter: LogicalFilter): RelNode = { // visit children and update inputs val input = filter.getInput.accept(this) + val materializer = createMaterializer(input) - // We do not materialize time indicators in conditions because they can be locally evaluated. - // Some conditions are evaluated by special operators (e.g., time window joins). - // Time indicators in remaining conditions are materialized by Calc before the code generation. - LogicalFilter.create(input, filter.getCondition) + val condition = filter.getCondition.accept(materializer) + LogicalFilter.create(input, condition) } override def visit(project: LogicalProject): RelNode = { // visit children and update inputs val input = project.getInput.accept(this) - - // check if input field contains time indicator type - // materialize field if no time indicator is present anymore - // if input field is already materialized, change to timestamp type - val materializer = new RexTimeIndicatorMaterializer( - rexBuilder, - input.getRowType.getFieldList.map(_.getType)) + val materializer = createMaterializer(input) val projects = project.getProjects.map(_.accept(materializer)) val fieldNames = project.getRowType.getFieldNames @@ -206,8 +194,14 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { override def visit(join: LogicalJoin): RelNode = { val left = join.getLeft.accept(this) val right = join.getRight.accept(this) + val materializer = createMaterializer(left, right) - LogicalJoin.create(left, right, join.getCondition, join.getVariablesSet, join.getJoinType) + LogicalJoin.create( + left, + right, + join.getCondition.accept(materializer), + join.getVariablesSet, + join.getJoinType) } def visit(temporalJoin: LogicalTemporalTableJoin): RelNode = { @@ -229,19 +223,11 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { case scan: LogicalTableFunctionScan => // visit children and update inputs val scanInputs = scan.getInputs.map(_.accept(this)) - - // check if input field contains time indicator type - // materialize field if no time indicator is present anymore - // if input field is already materialized, change to timestamp type - val materializer = new RexTimeIndicatorMaterializer( - rexBuilder, - inputs.head.getRowType.getFieldList.map(_.getType)) - - val call = scan.getCall.accept(materializer) + val materializer = createMaterializer(inputs.head) LogicalTableFunctionScan.create( scan.getCluster, scanInputs, - call, + scan.getCall.accept(materializer), scan.getElementType, scan.getRowType, scan.getColumnMappings) @@ -369,6 +355,15 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { indicesToMaterialize.toSet } + + private def createMaterializer(inputs: RelNode*): RexTimeIndicatorMaterializer = { + // check if input field contains time indicator type + // materialize field if no time indicator is present anymore + // if input field is already materialized, change to timestamp type + new RexTimeIndicatorMaterializer( + rexBuilder, + inputs.flatMap(_.getRowType.getFieldList.map(_.getType))) + } } object RelTimeIndicatorConverter { @@ -412,11 +407,34 @@ object RelTimeIndicatorConverter { * @return The expression with materialized time indicators. */ def convertExpression(expr: RexNode, rowType: RelDataType, rexBuilder: RexBuilder): RexNode = { + // check if input field contains time indicator type + // materialize field if no time indicator is present anymore + // if input field is already materialized, change to timestamp type val materializer = new RexTimeIndicatorMaterializer( - rexBuilder, - rowType.getFieldList.map(_.getType)) + rexBuilder, + rowType.getFieldList.map(_.getType)) + + expr.accept(materializer) + } + + /** + * Checks if the given call is a materialization call for either proctime or rowtime. + */ + def isMaterializationCall(call: RexCall): Boolean = { + val isProctimeCall: Boolean = { + call.getOperator == ProctimeSqlFunction && + call.getOperands.size() == 1 && + isProctimeIndicatorType(call.getOperands.get(0).getType) + } + + val isRowtimeCall: Boolean = { + call.getOperator == SqlStdOperatorTable.CAST && + call.getOperands.size() == 1 && + isRowtimeIndicatorType(call.getOperands.get(0).getType) && + call.getType.getSqlTypeName == SqlTypeName.TIMESTAMP + } - expr.accept(materializer) + isProctimeCall || isRowtimeCall } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala index 45414ee3ff6e7..ce57070ac98d7 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala @@ -270,8 +270,8 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc * Explicitly declares this external table for supporting only batch environments. */ def supportsBatch(): ExternalCatalogTableBuilder = { - isBatch = false - isStreaming = true + isBatch = true + isStreaming = false this } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala index 566e3d7cbc59a..57cc815fee087 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala @@ -142,6 +142,21 @@ class AggregationCodeGenerator( fields.mkString(", ") } + val parametersCodeForDistinctMerge = aggFields.map { inFields => + val fields = inFields.filter(_ > -1).zipWithIndex.map { case (f, i) => + // index to constant + if (f >= physicalInputTypes.length) { + constantFields(f - physicalInputTypes.length) + } + // index to input field + else { + s"(${CodeGenUtils.boxedTypeTermForTypeInfo(physicalInputTypes(f))}) k.getField($i)" + } + } + + fields.mkString(", ") + } + // get method signatures val classes = UserDefinedFunctionUtils.typeInfoToClass(physicalInputTypes) val constantClasses = UserDefinedFunctionUtils.typeInfoToClass(constantTypes) @@ -643,7 +658,7 @@ class AggregationCodeGenerator( | (${classOf[Row].getCanonicalName}) entry.getKey(); | Long v = (Long) entry.getValue(); | if (aDistinctAcc$i.add(k, v)) { - | ${aggs(i)}.accumulate(aAcc$i, k); + | ${aggs(i)}.accumulate(aAcc$i, ${parametersCodeForDistinctMerge(i)}); | } | } | a.setField($i, aDistinctAcc$i); diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala index 13bf50a5ef227..fdb0d509554f8 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala @@ -108,39 +108,39 @@ abstract class CodeGenerator( // set of member statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableMemberStatements: mutable.LinkedHashSet[String] = + val reusableMemberStatements: mutable.LinkedHashSet[String] = mutable.LinkedHashSet[String]() // set of constructor statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableInitStatements: mutable.LinkedHashSet[String] = + val reusableInitStatements: mutable.LinkedHashSet[String] = mutable.LinkedHashSet[String]() // set of open statements for RichFunction that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableOpenStatements: mutable.LinkedHashSet[String] = + val reusableOpenStatements: mutable.LinkedHashSet[String] = mutable.LinkedHashSet[String]() // set of close statements for RichFunction that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableCloseStatements: mutable.LinkedHashSet[String] = + val reusableCloseStatements: mutable.LinkedHashSet[String] = mutable.LinkedHashSet[String]() // set of statements that will be added only once per record; // code should only update member variables because local variables are not accessible if // the code needs to be split; // we use a LinkedHashSet to keep the insertion order - protected val reusablePerRecordStatements: mutable.LinkedHashSet[String] = + val reusablePerRecordStatements: mutable.LinkedHashSet[String] = mutable.LinkedHashSet[String]() // map of initial input unboxing expressions that will be added only once // (inputTerm, index) -> expr - protected val reusableInputUnboxingExprs: mutable.Map[(String, Int), GeneratedExpression] = + val reusableInputUnboxingExprs: mutable.Map[(String, Int), GeneratedExpression] = mutable.Map[(String, Int), GeneratedExpression]() // set of constructor statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableConstructorStatements: mutable.LinkedHashSet[(String, String)] = + val reusableConstructorStatements: mutable.LinkedHashSet[(String, String)] = mutable.LinkedHashSet[(String, String)]() /** @@ -1061,13 +1061,13 @@ abstract class CodeGenerator( // declaration val resultTypeTerm = primitiveTypeTermForTypeInfo(expr.resultType) - if (nullCheck) { + if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) { reusableMemberStatements.add(s"private boolean ${expr.nullTerm};") } reusableMemberStatements.add(s"private $resultTypeTerm ${expr.resultTerm};") // assignment - if (nullCheck) { + if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) { reusablePerRecordStatements.add(s"this.${expr.nullTerm} = ${expr.nullTerm};") } reusablePerRecordStatements.add(s"this.${expr.resultTerm} = ${expr.resultTerm};") diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala index 85d858fb75b99..ddc13c5b45b23 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala @@ -59,13 +59,14 @@ class CollectorCodeGenerator( * valid Java class identifier. * @param bodyCode body code for the collector method * @param collectedType The type information of the element collected by the collector + * @param filterGenerator generator containing context information for the generated body code * @return instance of GeneratedCollector */ def generateTableFunctionCollector( name: String, bodyCode: String, collectedType: TypeInformation[Any], - codeGenerator: CodeGenerator) + filterGenerator: CodeGenerator) : GeneratedCollector = { val className = newName(name) @@ -86,6 +87,10 @@ class CollectorCodeGenerator( s"$input2TypeClass $input2Term" // local variable } + reusableMemberStatements ++= filterGenerator.reusableMemberStatements + reusableInitStatements ++= filterGenerator.reusableInitStatements + reusablePerRecordStatements ++= filterGenerator.reusablePerRecordStatements + val funcCode = j""" |public class $className extends ${classOf[TableFunctionCollector[_]].getCanonicalName} { | @@ -98,7 +103,7 @@ class CollectorCodeGenerator( | | @Override | public void open(${classOf[Configuration].getCanonicalName} parameters) throws Exception { - | ${codeGenerator.reuseOpenCode()} + | ${filterGenerator.reuseOpenCode()} | } | | @Override @@ -113,7 +118,7 @@ class CollectorCodeGenerator( | | @Override | public void close() throws Exception { - | ${codeGenerator.reuseCloseCode()} + | ${filterGenerator.reuseCloseCode()} | } |} |""".stripMargin diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala index 282b167e91429..5a185da4922f5 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala @@ -839,8 +839,15 @@ object ScalarOperators { generateArithmeticOperator(op, nullCheck, l, left, right, config) case (SqlTimeTypeInfo.DATE, TimeIntervalTypeInfo.INTERVAL_MILLIS) => - generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.DATE, left, right) { - (l, r) => s"$l $op ((int) ($r / ${MILLIS_PER_DAY}L))" + resultType match { + case SqlTimeTypeInfo.DATE => + generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.DATE, left, right) { + (l, r) => s"$l $op ((int) ($r / ${MILLIS_PER_DAY}L))" + } + case SqlTimeTypeInfo.TIMESTAMP => + generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.TIMESTAMP, left, right) { + (l, r) => s"$l * ${MILLIS_PER_DAY}L $op $r" + } } case (SqlTimeTypeInfo.DATE, TimeIntervalTypeInfo.INTERVAL_MONTHS) => diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala index 9a10c118c0b65..246af6c0dab33 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala @@ -33,13 +33,14 @@ import org.apache.flink.table.api.dataview.ListView * @param listSerializer List serializer. * @tparam T The type of element in the list. */ -class ListViewSerializer[T](val listSerializer: ListSerializer[T]) +@SerialVersionUID(-2030398712359267867L) +class ListViewSerializer[T](val listSerializer: TypeSerializer[java.util.List[T]]) extends TypeSerializer[ListView[T]] { override def isImmutableType: Boolean = false override def duplicate(): TypeSerializer[ListView[T]] = { - new ListViewSerializer[T](listSerializer.duplicate().asInstanceOf[ListSerializer[T]]) + new ListViewSerializer[T](listSerializer.duplicate()) } override def createInstance(): ListView[T] = { @@ -75,41 +76,28 @@ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) override def equals(obj: Any): Boolean = canEqual(this) && listSerializer.equals(obj.asInstanceOf[ListViewSerializer[_]].listSerializer) - override def snapshotConfiguration(): ListViewSerializerConfigSnapshot[T] = - new ListViewSerializerConfigSnapshot[T](listSerializer) + override def snapshotConfiguration(): ListViewSerializerSnapshot[T] = + new ListViewSerializerSnapshot[T](listSerializer) override def ensureCompatibility( configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[ListView[T]] = { configSnapshot match { - case snapshot: ListViewSerializerConfigSnapshot[T] => - val previousListSerializerAndConfig = - snapshot.getSingleNestedSerializerAndConfig - - val compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousListSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousListSerializerAndConfig.f1, - listSerializer) - - if (!compatResult.isRequiresMigration) { - CompatibilityResult.compatible[ListView[T]] - } else { - CompatibilityResult.requiresMigration[ListView[T]] - } - // backwards compatibility path; - // Flink versions older or equal to 1.5.x returns a + // Flink versions older or equal to 1.6.x returns a // CollectionSerializerConfigSnapshot as the snapshot case legacySnapshot: CollectionSerializerConfigSnapshot[java.util.List[T], T] => val previousListSerializerAndConfig = legacySnapshot.getSingleNestedSerializerAndConfig + // in older versions, the nested list serializer was always + // specifically a ListSerializer, so this cast is safe + val castedSer = listSerializer.asInstanceOf[ListSerializer[T]] val compatResult = CompatibilityUtil.resolveCompatibilityResult( previousListSerializerAndConfig.f0, classOf[UnloadableDummyTypeSerializer[_]], previousListSerializerAndConfig.f1, - listSerializer.getElementSerializer) + castedSer.getElementSerializer) if (!compatResult.isRequiresMigration) { CompatibilityResult.compatible[ListView[T]] @@ -120,4 +108,6 @@ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) case _ => CompatibilityResult.requiresMigration[ListView[T]] } } + + def getListSerializer: TypeSerializer[java.util.List[T]] = listSerializer } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala index d6419e81ab351..89cdf70174983 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala @@ -35,14 +35,14 @@ import org.apache.flink.table.api.dataview.MapView * @tparam K The type of the keys in the map. * @tparam V The type of the values in the map. */ -class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) +@SerialVersionUID(-9007142882049098705L) +class MapViewSerializer[K, V](val mapSerializer: TypeSerializer[java.util.Map[K, V]]) extends TypeSerializer[MapView[K, V]] { override def isImmutableType: Boolean = false override def duplicate(): TypeSerializer[MapView[K, V]] = - new MapViewSerializer[K, V]( - mapSerializer.duplicate().asInstanceOf[MapSerializer[K, V]]) + new MapViewSerializer[K, V](mapSerializer.duplicate()) override def createInstance(): MapView[K, V] = { new MapView[K, V]() @@ -77,30 +77,14 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) override def equals(obj: Any): Boolean = canEqual(this) && mapSerializer.equals(obj.asInstanceOf[MapViewSerializer[_, _]].mapSerializer) - override def snapshotConfiguration(): MapViewSerializerConfigSnapshot[K, V] = - new MapViewSerializerConfigSnapshot[K, V](mapSerializer) + override def snapshotConfiguration(): MapViewSerializerSnapshot[K, V] = + new MapViewSerializerSnapshot[K, V](mapSerializer) // copy and modified from MapSerializer.ensureCompatibility override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot[_]) : CompatibilityResult[MapView[K, V]] = { configSnapshot match { - case snapshot: MapViewSerializerConfigSnapshot[K, V] => - val previousKvSerializersAndConfigs = - snapshot.getNestedSerializersAndConfigs - - val mapSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousKvSerializersAndConfigs.get(0).f1, - mapSerializer) - - if (!mapSerializerCompatResult.isRequiresMigration) { - CompatibilityResult.compatible[MapView[K, V]] - } else { - CompatibilityResult.requiresMigration[MapView[K, V]] - } - // backwards compatibility path; // Flink versions older or equal to 1.5.x returns a // MapSerializerConfigSnapshot as the snapshot @@ -108,17 +92,20 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) val previousKvSerializersAndConfigs = legacySnapshot.getNestedSerializersAndConfigs + // in older versions, the nested map serializer was always + // specifically a MapSerializer, so this cast is safe + val castedSer = mapSerializer.asInstanceOf[MapSerializer[K, V]] val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousKvSerializersAndConfigs.get(0).f0, classOf[UnloadableDummyTypeSerializer[_]], previousKvSerializersAndConfigs.get(0).f1, - mapSerializer) + castedSer.getKeySerializer) val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousKvSerializersAndConfigs.get(1).f0, classOf[UnloadableDummyTypeSerializer[_]], previousKvSerializersAndConfigs.get(1).f1, - mapSerializer.getValueSerializer) + castedSer.getValueSerializer) if (!keyCompatResult.isRequiresMigration && !valueCompatResult.isRequiresMigration) { CompatibilityResult.compatible[MapView[K, V]] @@ -129,4 +116,6 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) case _ => CompatibilityResult.requiresMigration[MapView[K, V]] } } + + def getMapSerializer: TypeSerializer[java.util.Map[K, V]] = mapSerializer } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala index 0b84f9490a135..0465e37f51876 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala @@ -303,7 +303,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers { case expr ~ _ ~ (MONTHS.key | MONTH.key) => toMonthInterval(expr, 1) - case expr ~ _ ~ (WEEKS.key | WEEKS.key) => toMilliInterval(expr, 7 * MILLIS_PER_DAY) + case expr ~ _ ~ (WEEKS.key | WEEK.key) => toMilliInterval(expr, 7 * MILLIS_PER_DAY) case expr ~ _ ~ (DAYS.key | DAY.key) => toMilliInterval(expr, MILLIS_PER_DAY) @@ -455,7 +455,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers { // suffix/prefix composite - lazy val composite: PackratParser[Expression] = over | nullLiteral | suffixed | prefixed | atom | + lazy val composite: PackratParser[Expression] = over | suffixed | nullLiteral | prefixed | atom | failure("Composite expression expected.") // unary ops diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala index c77bd7a41ba17..f6ea4bd8c28cf 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala @@ -56,7 +56,7 @@ abstract sealed class Aggregation extends Expression { case class DistinctAgg(child: Expression) extends Aggregation { - private[flink] def distinct: Expression = DistinctAgg(child) + def distinct: Expression = DistinctAgg(child) override private[flink] def resultType: TypeInformation[_] = child.resultType diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala index 6fd1f7afb16b5..0b685239e05aa 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala @@ -23,7 +23,7 @@ import java.util import org.apache.flink.table.sinks.BatchTableSink /** - * A factory to create configured table sink instances in a streaming environment based on + * A factory to create configured table sink instances in a batch environment based on * string-based properties. See also [[TableFactory]] for more information. * * @tparam T type of records that the factory consumes diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala index c75e6faf1077e..bd221c208fb98 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala @@ -28,7 +28,7 @@ import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{getAccum private[flink] case class DistinctAggregateFunction[T: TypeInformation, ACC: TypeInformation] (aggFunction: AggregateFunction[T, ACC]) { - private[flink] def distinct(params: Expression*): Expression = { + def distinct(params: Expression*): Expression = { val resultTypeInfo: TypeInformation[_] = getResultTypeOfAggregateFunction( aggFunction, implicitly[TypeInformation[T]]) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala index 84e3f79cec3e2..c0cfa247bca86 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala @@ -633,6 +633,8 @@ case class WindowAggregate( case aggExpr: Aggregation if aggExpr.getSqlAggFunction.requiresOver => failValidation(s"OVER clause is necessary for window functions: [${aggExpr.getClass}].") + case aggExpr: DistinctAgg => + validateAggregateExpression(aggExpr.child) // check no nested aggregation exists. case aggExpr: Aggregation => aggExpr.children.foreach { child => diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala index 53adf6b5d309e..b9b5a3e7fd56e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.plan.nodes import org.apache.calcite.rel.`type`.RelDataType -import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle} +import org.apache.calcite.rex.{RexCall, RexNode} import org.apache.calcite.sql.SemiJoinType import org.apache.flink.api.common.functions.Function import org.apache.flink.api.common.typeinfo.TypeInformation @@ -149,18 +149,9 @@ trait CommonCorrelate { |getCollector().collect(${crossResultExpr.resultTerm}); |""".stripMargin } else { - - // adjust indices of InputRefs to adhere to schema expected by generator - val changeInputRefIndexShuttle = new RexShuttle { - override def visitInputRef(inputRef: RexInputRef): RexNode = { - new RexInputRef(inputSchema.arity + inputRef.getIndex, inputRef.getType) - } - } - // Run generateExpression to add init statements (ScalarFunctions) of condition to generator. - // The generated expression is discarded. - generator.generateExpression(condition.get.accept(changeInputRefIndexShuttle)) - filterGenerator.input1Term = filterGenerator.input2Term + // generating filter expressions might add init statements and member fields + // that need to be combined with the statements of the collector code generator val filterCondition = filterGenerator.generateExpression(condition.get) s""" |${filterGenerator.reuseInputUnboxingCode()} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala index 6e2ccdeba5b04..5e0ee32ad6c5d 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala @@ -39,18 +39,14 @@ object FlinkRuleSets { SubQueryRemoveRule.JOIN) /** - * Handles proper conversion of correlate queries with temporal table functions - * into temporal table joins. This can create new table scans in the plan. + * Expand plan by replacing references to tables into a proper plan sub trees. Those rules + * can create new plan nodes. */ - val TEMPORAL_JOIN_RULES: RuleSet = RuleSets.ofList( - LogicalCorrelateToTemporalTableJoinRule.INSTANCE - ) + val EXPAND_PLAN_RULES: RuleSet = RuleSets.ofList( + LogicalCorrelateToTemporalTableJoinRule.INSTANCE, + TableScanRule.INSTANCE) - /** - * Convert table references before query decorrelation. - */ - val TABLE_REF_RULES: RuleSet = RuleSets.ofList( - TableScanRule.INSTANCE, + val POST_EXPAND_CLEAN_UP_RULES: RuleSet = RuleSets.ofList( EnumerableToLogicalTableScan.INSTANCE) val LOGICAL_OPT_RULES: RuleSet = RuleSets.ofList( diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala index 072acb34c7309..ad749d25df32f 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala @@ -20,10 +20,8 @@ package org.apache.flink.table.plan.rules.datastream import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode -import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.convert.ConverterRule -import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode} -import org.apache.flink.table.api.TableConfig +import org.apache.flink.table.api.{TableConfig, TableException} import org.apache.flink.table.calcite.FlinkTypeFactory import org.apache.flink.table.plan.nodes.FlinkConventions import org.apache.flink.table.plan.nodes.datastream.DataStreamJoin @@ -40,45 +38,32 @@ class DataStreamJoinRule FlinkConventions.DATASTREAM, "DataStreamJoinRule") { - /** - * Checks if an expression accesses a time attribute. - * - * @param expr The expression to check. - * @param inputType The input type of the expression. - * @return True, if the expression accesses a time attribute. False otherwise. - */ - def accessesTimeAttribute(expr: RexNode, inputType: RelDataType): Boolean = { - expr match { - case i: RexInputRef => - val accessedType = inputType.getFieldList.get(i.getIndex).getType - FlinkTypeFactory.isTimeIndicatorType(accessedType) - case c: RexCall => - c.operands.asScala.exists(accessesTimeAttribute(_, inputType)) - case _ => false - } - } - override def matches(call: RelOptRuleCall): Boolean = { val join: FlinkLogicalJoin = call.rel(0).asInstanceOf[FlinkLogicalJoin] val joinInfo = join.analyzeCondition - val (windowBounds, remainingPreds) = WindowJoinUtil.extractWindowBoundsFromPredicate( + val (windowBounds, _) = WindowJoinUtil.extractWindowBoundsFromPredicate( joinInfo.getRemaining(join.getCluster.getRexBuilder), join.getLeft.getRowType.getFieldCount, join.getRowType, join.getCluster.getRexBuilder, TableConfig.DEFAULT) - // remaining predicate must not access time attributes - val remainingPredsAccessTime = remainingPreds.isDefined && - accessesTimeAttribute(remainingPreds.get, join.getRowType) + if (windowBounds.isDefined) { + return false + } - // Check that no event-time attributes are in the input because non-window join is unbounded - // and we don't know how much to hold back watermarks. + // Check that no event-time attributes are in the outputs (composed of two inputs) + // because non-window join is unbounded and we don't know how much to hold back watermarks. val rowTimeAttrInOutput = join.getRowType.getFieldList.asScala .exists(f => FlinkTypeFactory.isRowtimeIndicatorType(f.getType)) - windowBounds.isEmpty && !remainingPredsAccessTime && !rowTimeAttrInOutput + if (rowTimeAttrInOutput) { + throw new TableException( + "Rowtime attributes must not be in the input rows of a regular join. " + + "As a workaround you can cast the time attributes of input tables to TIMESTAMP before.") + } + true } override def convert(rel: RelNode): RelNode = { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala index 62eecbb053217..c8c58adf22682 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala @@ -22,7 +22,7 @@ import java.math.{BigDecimal => JBigDecimal} import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rex._ -import org.apache.calcite.sql.`type`.SqlTypeName +import org.apache.calcite.sql.`type`.{SqlTypeFamily, SqlTypeName} import org.apache.flink.table.api.scala.{Session, Slide, Tumble} import org.apache.flink.table.api.{TableException, ValidationException, Window} import org.apache.flink.table.calcite.FlinkTypeFactory @@ -68,8 +68,10 @@ class DataStreamLogicalWindowAggregateRule def getOperandAsLong(call: RexCall, idx: Int): Long = call.getOperands.get(idx) match { - case v: RexLiteral => v.getValue.asInstanceOf[JBigDecimal].longValue() - case _ => throw new TableException("Only constant window descriptors are supported.") + case v: RexLiteral if v.getTypeName.getFamily == SqlTypeFamily.INTERVAL_DAY_TIME => + v.getValue.asInstanceOf[JBigDecimal].longValue() + case _ => throw new TableException( + "Only constant window intervals with millisecond resolution are supported.") } def getOperandAsTimeIndicator(call: RexCall, idx: Int): ResolvedFieldReference = diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala index 5b0aa65362f5a..5c0241f55e705 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala @@ -18,15 +18,23 @@ package org.apache.flink.table.plan.rules.datastream -import org.apache.calcite.plan.{RelOptRule, RelTraitSet} +import java.util.{List => JList} + +import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode +import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.convert.ConverterRule -import org.apache.flink.table.api.TableException +import org.apache.calcite.rex.{RexInputRef, RexNode} +import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.plan.logical.MatchRecognize import org.apache.flink.table.plan.nodes.FlinkConventions import org.apache.flink.table.plan.nodes.datastream.DataStreamMatch import org.apache.flink.table.plan.nodes.logical.FlinkLogicalMatch import org.apache.flink.table.plan.schema.RowSchema +import org.apache.flink.table.plan.util.RexDefaultVisitor + +import scala.collection.JavaConverters._ +import scala.collection.mutable class DataStreamMatchRule extends ConverterRule( @@ -35,6 +43,14 @@ class DataStreamMatchRule FlinkConventions.DATASTREAM, "DataStreamMatchRule") { + override def matches(call: RelOptRuleCall): Boolean = { + val logicalMatch: FlinkLogicalMatch = call.rel(0).asInstanceOf[FlinkLogicalMatch] + + // This check might be obsolete once CALCITE-2747 is resolved + validateAmbiguousColumns(logicalMatch) + true + } + override def convert(rel: RelNode): RelNode = { val logicalMatch: FlinkLogicalMatch = rel.asInstanceOf[FlinkLogicalMatch] val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.DATASTREAM) @@ -71,6 +87,47 @@ class DataStreamMatchRule new RowSchema(logicalMatch.getRowType), new RowSchema(logicalMatch.getInput.getRowType)) } + + private def validateAmbiguousColumns(logicalMatch: FlinkLogicalMatch): Unit = { + if (logicalMatch.isAllRows) { + throw new TableException("All rows per match mode is not supported yet.") + } else { + val refNameFinder = new RefNameFinder(logicalMatch.getInput.getRowType) + validateAmbiguousColumnsOnRowPerMatch( + logicalMatch.getPartitionKeys, + logicalMatch.getMeasures.keySet().asScala, + logicalMatch.getRowType, + refNameFinder) + } + } + + private def validateAmbiguousColumnsOnRowPerMatch( + partitionKeys: JList[RexNode], + measuresNames: mutable.Set[String], + expectedSchema: RelDataType, + refNameFinder: RefNameFinder) + : Unit = { + val actualSize = partitionKeys.size() + measuresNames.size + val expectedSize = expectedSchema.getFieldCount + if (actualSize != expectedSize) { + //try to find ambiguous column + + val ambiguousColumns = partitionKeys.asScala.map(_.accept(refNameFinder)) + .filter(measuresNames.contains).mkString("{", ", ", "}") + + throw new ValidationException(s"Columns ambiguously defined: $ambiguousColumns") + } + } + + private class RefNameFinder(inputSchema: RelDataType) extends RexDefaultVisitor[String] { + + override def visitInputRef(inputRef: RexInputRef): String = { + inputSchema.getFieldList.get(inputRef.getIndex).getName + } + + override def visitNode(rexNode: RexNode): String = + throw new TableException(s"PARTITION BY clause accepts only input reference. Found $rexNode") + } } object DataStreamMatchRule { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala index 94ff19c229daf..5bdea53b58a27 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala @@ -22,12 +22,10 @@ import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode import org.apache.calcite.rel.convert.ConverterRule import org.apache.calcite.rel.core.JoinRelType -import org.apache.flink.table.api.TableConfig import org.apache.flink.table.plan.nodes.FlinkConventions import org.apache.flink.table.plan.nodes.datastream.DataStreamTemporalTableJoin import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTemporalTableJoin import org.apache.flink.table.plan.schema.RowSchema -import org.apache.flink.table.runtime.join.WindowJoinUtil class DataStreamTemporalTableJoinRule extends ConverterRule( @@ -38,16 +36,7 @@ class DataStreamTemporalTableJoinRule override def matches(call: RelOptRuleCall): Boolean = { val join: FlinkLogicalTemporalTableJoin = call.rel(0) - val joinInfo = join.analyzeCondition - - val (windowBounds, remainingPreds) = WindowJoinUtil.extractWindowBoundsFromPredicate( - joinInfo.getRemaining(join.getCluster.getRexBuilder), - join.getLeft.getRowType.getFieldCount, - join.getRowType, - join.getCluster.getRexBuilder, - TableConfig.DEFAULT) - - windowBounds.isEmpty && join.getJoinType == JoinRelType.INNER + join.getJoinType == JoinRelType.INNER } override def convert(rel: RelNode): RelNode = { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala index 3dfae99a69629..cd9c5a86f342c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala @@ -52,7 +52,7 @@ class DataStreamWindowJoinRule if (windowBounds.get.isEventTime) { true } else { - // Check that no event-time attributes are in the input because the processing time window + // Check that no event-time attributes are in the output because the processing time window // join does not correctly hold back watermarks. // We rely on projection pushdown to remove unused attributes before the join. !join.getRowType.getFieldList.asScala diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala index 54bac601bcc10..cb451b92cd7c0 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable @@ -46,6 +47,8 @@ class CRowMapRunner[OUT]( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating MapFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) } override def map(in: CRow): OUT = { @@ -53,4 +56,8 @@ class CRowMapRunner[OUT]( } override def getProducedType: TypeInformation[OUT] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala index 600b8987a28f4..ebef577ff8358 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable import org.apache.flink.configuration.Configuration @@ -49,6 +50,8 @@ class CRowOutputProcessRunner( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating ProcessFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) this.cRowWrapper = new CRowWrappingCollector() this.cRowWrapper.setChange(true) @@ -68,4 +71,8 @@ class CRowOutputProcessRunner( } override def getProducedType: TypeInformation[CRow] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala index 0bf65694367f9..26ed962e01a68 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{FlatJoinFunction, RichFlatJoinFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable @@ -42,10 +43,16 @@ class FlatJoinRunner[IN1, IN2, OUT]( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating FlatJoinFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) } override def join(first: IN1, second: IN2, out: Collector[OUT]): Unit = function.join(first, second, out) override def getProducedType: TypeInformation[OUT] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala index 00d18ecc00794..7f56ffe72cc62 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable @@ -41,10 +42,16 @@ class MapRunner[IN, OUT]( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating MapFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) } override def map(in: IN): OUT = function.map(in) override def getProducedType: TypeInformation[OUT] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala index 5f5a2cc4a19a3..f15aaa6432ee3 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{FlatJoinFunction, RichFlatMapFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable @@ -43,6 +44,8 @@ abstract class MapSideJoinRunner[IN1, IN2, SINGLE_IN, MULTI_IN, OUT]( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating FlatJoinFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) broadcastSet = retrieveBroadcastSet } @@ -56,4 +59,8 @@ abstract class MapSideJoinRunner[IN1, IN2, SINGLE_IN, MULTI_IN, OUT]( } override def getProducedType: TypeInformation[OUT] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/OutputRowtimeProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/OutputRowtimeProcessFunction.scala index 3eaeea308f915..c964a4b7165ae 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/OutputRowtimeProcessFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/OutputRowtimeProcessFunction.scala @@ -55,4 +55,7 @@ class OutputRowtimeProcessFunction[OUT]( out.collect(function.map(in)) } + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala index 397032003ec81..c59efe250c3d2 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala @@ -95,6 +95,13 @@ class GroupAggProcessFunction( var inputCnt = cntState.value() if (null == accumulators) { + // Don't create a new accumulator for a retraction message. This + // might happen if the retraction message is the first message for the + // key or after a state clean up. + if (!inputC.change) { + return + } + // first accumulate message firstRow = true accumulators = function.createAccumulators() } else { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala index 0fe2e39c3a718..c59f4c2f44b04 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime.join import org.apache.flink.api.common.functions.FlatJoinFunction +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state.{MapState, MapStateDescriptor, ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2} @@ -77,14 +78,16 @@ abstract class NonWindowJoin( protected var curProcessTime: Long = _ override def open(parameters: Configuration): Unit = { - LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n " + - s"Code:\n$genJoinFuncCode") + LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode") val clazz = compile( getRuntimeContext.getUserCodeClassLoader, genJoinFuncName, genJoinFuncCode) + LOG.debug("Instantiating JoinFunction.") joinFunction = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(joinFunction, getRuntimeContext) + FunctionUtils.openFunction(joinFunction, parameters) // initialize left and right state, the first element of tuple2 indicates how many rows of // this row, while the second element represents the expired time of this row. @@ -291,4 +294,8 @@ abstract class NonWindowJoin( joinFunction.join(otherSideRow, inputRow, cRowWrapper) } } + + override def close(): Unit = { + FunctionUtils.closeFunction(joinFunction) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala index 175af91e9e47d..7a8bb61b1cfce 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala @@ -18,8 +18,10 @@ package org.apache.flink.table.runtime.join import org.apache.flink.api.common.functions.FlatJoinFunction +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.operators.{AbstractStreamOperator, TimestampedCollector, TwoInputStreamOperator} import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.table.api.StreamQueryConfig @@ -51,12 +53,16 @@ class TemporalProcessTimeJoin( protected var joinFunction: FlatJoinFunction[Row, Row, Row] = _ override def open(): Unit = { + LOG.debug(s"Compiling FlatJoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode") val clazz = compile( getRuntimeContext.getUserCodeClassLoader, genJoinFuncName, genJoinFuncCode) + LOG.debug("Instantiating FlatJoinFunction.") joinFunction = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(joinFunction, getRuntimeContext) + FunctionUtils.openFunction(joinFunction, new Configuration()) val rightStateDescriptor = new ValueStateDescriptor[Row]("right", rightType) rightState = getRuntimeContext.getState(rightStateDescriptor) @@ -86,4 +92,8 @@ class TemporalProcessTimeJoin( rightState.clear() } } + + override def close(): Unit = { + FunctionUtils.closeFunction(joinFunction) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala index 17f8e1a233f10..66b60d44969fa 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala @@ -22,8 +22,10 @@ import java.util import java.util.{Comparator, Optional} import org.apache.flink.api.common.functions.FlatJoinFunction +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state._ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} +import org.apache.flink.configuration.Configuration import org.apache.flink.runtime.state.{VoidNamespace, VoidNamespaceSerializer} import org.apache.flink.streaming.api.SimpleTimerService import org.apache.flink.streaming.api.operators._ @@ -115,12 +117,16 @@ class TemporalRowtimeJoin( private var joinFunction: FlatJoinFunction[Row, Row, Row] = _ override def open(): Unit = { + LOG.debug(s"Compiling FlatJoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode") val clazz = compile( getRuntimeContext.getUserCodeClassLoader, genJoinFuncName, genJoinFuncCode) + LOG.debug("Instantiating FlatJoinFunction.") joinFunction = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(joinFunction, getRuntimeContext) + FunctionUtils.openFunction(joinFunction, new Configuration()) nextLeftIndex = getRuntimeContext.getState( new ValueStateDescriptor[JLong](NEXT_LEFT_INDEX_STATE_NAME, BasicTypeInfo.LONG_TYPE_INFO)) @@ -159,6 +165,22 @@ class TemporalRowtimeJoin( registerSmallestTimer(rowTime) // Timer to clean up the state } + override def onProcessingTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { + throw new IllegalStateException("This should never happen") + } + + override def onEventTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { + registeredTimer.clear() + val lastUnprocessedTime = emitResultAndCleanUpState(timerService.currentWatermark()) + if (lastUnprocessedTime < Long.MaxValue) { + registerTimer(lastUnprocessedTime) + } + } + + override def close(): Unit = { + FunctionUtils.closeFunction(joinFunction) + } + private def registerSmallestTimer(timestamp: Long): Unit = { val currentRegisteredTimer = registeredTimer.value() if (currentRegisteredTimer == null) { @@ -175,18 +197,6 @@ class TemporalRowtimeJoin( timerService.registerEventTimeTimer(timestamp) } - override def onProcessingTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { - throw new IllegalStateException("This should never happen") - } - - override def onEventTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { - registeredTimer.clear() - val lastUnprocessedTime = emitResultAndCleanUpState(timerService.currentWatermark()) - if (lastUnprocessedTime < Long.MaxValue) { - registerTimer(lastUnprocessedTime) - } - } - /** * @return a row time of the oldest unprocessed probe record or Long.MaxValue, if all records * have been processed. diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TimeBoundedStreamJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TimeBoundedStreamJoin.scala index a44bdef2e0907..ae7ca6a6032da 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TimeBoundedStreamJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TimeBoundedStreamJoin.scala @@ -22,6 +22,7 @@ import java.util import java.util.{List => JList} import org.apache.flink.api.common.functions.FlatJoinFunction +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state._ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} import org.apache.flink.api.java.operators.join.JoinType @@ -110,6 +111,8 @@ abstract class TimeBoundedStreamJoin( genJoinFuncCode) LOG.debug("Instantiating JoinFunction.") joinFunction = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(joinFunction, getRuntimeContext) + FunctionUtils.openFunction(joinFunction, config) joinCollector = new EmitAwareCollector() joinCollector.setCRowChange(true) @@ -378,6 +381,10 @@ abstract class TimeBoundedStreamJoin( } } + override def close(): Unit = { + FunctionUtils.closeFunction(joinFunction) + } + /** * Calculate the expiration time with the given operator time and relative window size. * diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/WindowJoinUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/WindowJoinUtil.scala index 18e26df89ccab..3e355e800a10a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/WindowJoinUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/WindowJoinUtil.scala @@ -23,12 +23,14 @@ import org.apache.calcite.plan.RelOptUtil import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.core.JoinRelType import org.apache.calcite.rex._ -import org.apache.calcite.sql.SqlKind +import org.apache.calcite.sql.fun.SqlStdOperatorTable +import org.apache.calcite.sql.{SqlKind, SqlOperatorTable} import org.apache.flink.api.common.functions.FlatJoinFunction import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.table.api.TableConfig -import org.apache.flink.table.calcite.FlinkTypeFactory +import org.apache.flink.table.calcite.{FlinkTypeFactory, RelTimeIndicatorConverter} import org.apache.flink.table.codegen.{ExpressionReducer, FunctionCodeGenerator, GeneratedFunction} +import org.apache.flink.table.functions.sql.ProctimeSqlFunction import org.apache.flink.table.plan.schema.{RowSchema, TimeIndicatorRelDataType} import org.apache.flink.types.Row @@ -380,13 +382,13 @@ object WindowJoinUtil { */ def replaceTimeFieldWithLiteral(expr: RexNode): RexNode = { expr match { + case c: RexCall if RelTimeIndicatorConverter.isMaterializationCall(c) => + // replace with timestamp + rexBuilder.makeZeroLiteral(expr.getType) case c: RexCall => // replace in call operands val newOps = c.operands.asScala.map(replaceTimeFieldWithLiteral).asJava rexBuilder.makeCall(c.getType, c.getOperator, newOps) - case i: RexInputRef if FlinkTypeFactory.isTimeIndicatorType(i.getType) => - // replace with timestamp - rexBuilder.makeZeroLiteral(expr.getType) case _ => expr } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/IterativeConditionRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/IterativeConditionRunner.scala index fc6755ea79fdd..15b9d37488948 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/IterativeConditionRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/IterativeConditionRunner.scala @@ -43,6 +43,7 @@ class IterativeConditionRunner( val clazz = compile(Thread.currentThread().getContextClassLoader, name, code) LOG.debug("Instantiating IterativeCondition.") function = clazz.newInstance() + // TODO add logic for opening and closing the function once it can be a RichFunction } override def filter(value: Row, ctx: IterativeCondition.Context[Row]): Boolean = { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/PatternSelectFunctionRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/PatternSelectFunctionRunner.scala index 643352271b5e3..c92cd38401215 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/PatternSelectFunctionRunner.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/PatternSelectFunctionRunner.scala @@ -48,6 +48,7 @@ class PatternSelectFunctionRunner( val clazz = compile(Thread.currentThread().getContextClassLoader, name, code) LOG.debug("Instantiating PatternSelectFunction.") function = clazz.newInstance() + // TODO add logic for opening and closing the function once it can be a RichFunction } override def flatSelect( diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinGroupReduceRunners.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinGroupReduceRunners.scala index 9b0f08ea4a137..23ad51a0effa8 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinGroupReduceRunners.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinGroupReduceRunners.scala @@ -19,6 +19,7 @@ package org.apache.flink.table.runtime import java.lang.Iterable +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{JoinFunction, RichGroupReduceFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration @@ -41,6 +42,12 @@ abstract class OuterJoinGroupReduceRunner( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating JoinFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, config) + } + + override def close(): Unit = { + FunctionUtils.closeFunction(function) } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinRunners.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinRunners.scala index a9e0211c40fc8..fd3b46fbf632c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinRunners.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/outerJoinRunners.scala @@ -20,6 +20,7 @@ package org.apache.flink.table.runtime import java.lang.{Boolean => JBool} +import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.functions.{JoinFunction, RichFlatJoinFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ResultTypeQueryable @@ -45,9 +46,15 @@ abstract class OuterJoinRunner( val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code) LOG.debug("Instantiating FlatJoinFunction.") function = clazz.newInstance() + FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext) + FunctionUtils.openFunction(function, parameters) } override def getProducedType: TypeInformation[Row] = returnType + + override def close(): Unit = { + FunctionUtils.closeFunction(function) + } } /** diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala index 0ce3aee3739e7..b3fe508515152 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala @@ -81,7 +81,7 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali // -------------------------------------------------------------------------------------------- override def snapshotConfiguration(): TypeSerializerConfigSnapshot[CRow] = { - new CRowSerializer.CRowSerializerConfigSnapshot(rowSerializer) + new CRowSerializer.CRowSerializerConfigSnapshot(Array(rowSerializer)) } override def ensureCompatibility( @@ -115,9 +115,13 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali object CRowSerializer { - class CRowSerializerConfigSnapshot(rowSerializers: TypeSerializer[Row]*) + class CRowSerializerConfigSnapshot(rowSerializers: Array[TypeSerializer[Row]]) extends CompositeTypeSerializerConfigSnapshot[CRow](rowSerializers: _*) { + def this() { + this(Array.empty) + } + override def getVersion: Int = CRowSerializerConfigSnapshot.VERSION } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala index 15d68900d219d..998fc1ff34083 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala @@ -114,12 +114,12 @@ object TableSourceUtil { val rowtimeIdx = schema.getFieldNames.indexOf(rowtimeAttribute) // ensure that field exists if (rowtimeIdx < 0) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a rowtime attribute for field " + s"'$rowtimeAttribute' but field '$rowtimeAttribute' does not exist in table.") } // ensure that field is of type TIMESTAMP if (schema.getFieldTypes()(rowtimeIdx) != Types.SQL_TIMESTAMP) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a rowtime attribute for field " + s"'$rowtimeAttribute' but field '$rowtimeAttribute' is not of type TIMESTAMP.") } // look up extractor input fields in return type @@ -138,12 +138,12 @@ object TableSourceUtil { val proctimeIdx = schema.getFieldNames.indexOf(proctimeAttribute) // ensure that field exists if (proctimeIdx < 0) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a processing time attribute for field " + s"'$proctimeAttribute' but field '$proctimeAttribute' does not exist in table.") } // ensure that field is of type TIMESTAMP if (schema.getFieldTypes()(proctimeIdx) != Types.SQL_TIMESTAMP) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a processing time attribute for field " + s"'$proctimeAttribute' but field '$proctimeAttribute' is not of type TIMESTAMP.") } case _ => // nothing to validate diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/ListViewSerializerSnapshotMigrationTest.java similarity index 52% rename from flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java rename to flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/ListViewSerializerSnapshotMigrationTest.java index b2126028de2a0..5465ada23ff28 100644 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java +++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/ListViewSerializerSnapshotMigrationTest.java @@ -18,29 +18,28 @@ package org.apache.flink.table.dataview; -import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase; import org.apache.flink.api.common.typeutils.base.ListSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.table.api.dataview.ListView; /** - * A {@link TypeSerializerConfigSnapshot} for the {@link ListViewSerializer}. - * - * @param the type of the list elements. + * Migration test for the {@link ListViewSerializerSnapshot}. */ -public final class ListViewSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - - private static final int VERSION = 1; +public class ListViewSerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase> { - /** This empty nullary constructor is required for deserializing the configuration. */ - public ListViewSerializerConfigSnapshot() {} - - public ListViewSerializerConfigSnapshot(ListSerializer listSerializer) { - super(listSerializer); - } + private static final String DATA = "flink-1.6-list-view-serializer-data"; + private static final String SNAPSHOT = "flink-1.6-list-view-serializer-snapshot"; - @Override - public int getVersion() { - return VERSION; + public ListViewSerializerSnapshotMigrationTest() { + super( + TestSpecification.>builder( + "1.6-list-view-serializer", + ListViewSerializer.class, + ListViewSerializerSnapshot.class) + .withSerializerProvider(() -> new ListViewSerializer<>(new ListSerializer<>(StringSerializer.INSTANCE))) + .withSnapshotDataLocation(SNAPSHOT) + .withTestData(DATA, 10) + ); } } diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/MapViewSerializerSnapshotMigrationTest.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/MapViewSerializerSnapshotMigrationTest.java new file mode 100644 index 0000000000000..66c7f17347456 --- /dev/null +++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/dataview/MapViewSerializerSnapshotMigrationTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataview; + +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.common.typeutils.base.MapSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.table.api.dataview.MapView; + +/** + * Migration test for the {@link MapViewSerializerSnapshot}. + */ +public class MapViewSerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase> { + + private static final String DATA = "flink-1.6-map-view-serializer-data"; + private static final String SNAPSHOT = "flink-1.6-map-view-serializer-snapshot"; + + public MapViewSerializerSnapshotMigrationTest() { + super( + TestSpecification.>builder( + "1.6-map-view-serializer", + MapViewSerializer.class, + MapViewSerializerSnapshot.class) + .withSerializerProvider(() -> new MapViewSerializer<>( + new MapSerializer<>(IntSerializer.INSTANCE, StringSerializer.INSTANCE))) + .withSnapshotDataLocation(SNAPSHOT) + .withTestData(DATA, 10) + ); + } +} diff --git a/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-data b/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-data new file mode 100644 index 0000000000000..7b6c68ac7f38e Binary files /dev/null and b/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-data differ diff --git a/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-snapshot b/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-snapshot new file mode 100644 index 0000000000000..2e2fd8ab9f10a Binary files /dev/null and b/flink-libraries/flink-table/src/test/resources/flink-1.6-list-view-serializer-snapshot differ diff --git a/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-data b/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-data new file mode 100644 index 0000000000000..6d68f192c0355 Binary files /dev/null and b/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-data differ diff --git a/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-snapshot b/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-snapshot new file mode 100644 index 0000000000000..9546a160ec0dc Binary files /dev/null and b/flink-libraries/flink-table/src/test/resources/flink-1.6-map-view-serializer-snapshot differ diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala index 736f9a2cad0c1..37d5bc1ea9795 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala @@ -21,6 +21,7 @@ import org.apache.calcite.rel.logical.LogicalJoin import org.apache.flink.api.scala._ import org.apache.flink.table.api.Types import org.apache.flink.table.api.scala._ +import org.apache.flink.table.calcite.RelTimeIndicatorConverter import org.apache.flink.table.expressions.Null import org.apache.flink.table.plan.logical.TumblingGroupWindow import org.apache.flink.table.runtime.join.WindowJoinUtil @@ -29,6 +30,9 @@ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Assert._ import org.junit.Test +/** + * Tests for both windowed and non-windowed joins. + */ class JoinTest extends TableTestBase { private val streamUtil: StreamTableTestUtil = streamTestUtil() streamUtil.addTable[(Int, String, Long)]("MyTable", 'a, 'b, 'c.rowtime, 'proctime.proctime) @@ -62,8 +66,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "proctime") ), term("where", - "AND(=(a, a0), >=(proctime, -(proctime0, 3600000)), " + - "<=(proctime, +(proctime0, 3600000)))"), + "AND(=(a, a0), >=(PROCTIME(proctime), -(PROCTIME(proctime0), 3600000)), " + + "<=(PROCTIME(proctime), +(PROCTIME(proctime0), 3600000)))"), term("join", "a, proctime, a0, b, proctime0"), term("joinType", "InnerJoin") ), @@ -100,8 +104,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 10000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 10000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, c, a0, b, c0"), term("joinType", "InnerJoin") ), @@ -138,8 +142,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "proctime") ), term("where", - "AND(=(a, a0), >=(proctime, -(proctime0, 3600000)), " + - "<=(proctime, +(proctime0, 3600000)))"), + "AND(=(a, a0), >=(PROCTIME(proctime), -(PROCTIME(proctime0), 3600000)), " + + "<=(PROCTIME(proctime), +(PROCTIME(proctime0), 3600000)))"), term("join", "a, proctime, a0, b, proctime0"), term("joinType", "InnerJoin") ), @@ -176,8 +180,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 600000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 600000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, c, a0, b, c0"), term("joinType", "InnerJoin") ), @@ -208,7 +212,7 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "a", "b", "proctime") ), - term("where", "AND(=(a, a0), =(proctime, proctime0))"), + term("where", "AND(=(a, a0), =(PROCTIME(proctime), PROCTIME(proctime0)))"), term("join", "a", "proctime", "a0", "b", "proctime0"), term("joinType", "InnerJoin") ), @@ -238,7 +242,7 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "a", "b", "c") ), - term("where", "AND(=(a, a0), =(c, c0))"), + term("where", "AND(=(a, a0), =(CAST(c), CAST(c0)))"), term("join", "a", "c", "a0", "b", "c0"), term("joinType", "InnerJoin") ), @@ -280,8 +284,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "a", "c", "proctime", "12 AS nullField") ), - term("where", "AND(=(a, a0), =(nullField, nullField0), >=(proctime, " + - "-(proctime0, 5000)), <=(proctime, +(proctime0, 5000)))"), + term("where", "AND(=(a, a0), =(nullField, nullField0), >=(PROCTIME(proctime), " + + "-(PROCTIME(proctime0), 5000)), <=(PROCTIME(proctime), +(PROCTIME(proctime0), 5000)))"), term("join", "a", "c", "proctime", "nullField", "a0", "c0", "proctime0", "nullField0"), term("joinType", "InnerJoin") ), @@ -320,8 +324,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 600000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 600000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, b, c, a0, b0, c0"), term("joinType", "InnerJoin") ), @@ -365,8 +369,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 600000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 600000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, b, c, a0, b0, c0"), term("joinType", "InnerJoin") ), @@ -408,8 +412,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "proctime") ), term("where", - "AND(=(a, a0), >=(proctime, -(proctime0, 3600000)), " + - "<=(proctime, +(proctime0, 3600000)))"), + "AND(=(a, a0), >=(PROCTIME(proctime), -(PROCTIME(proctime0), 3600000)), " + + "<=(PROCTIME(proctime), +(PROCTIME(proctime0), 3600000)))"), term("join", "a, proctime, a0, b, proctime0"), term("joinType", "LeftOuterJoin") ), @@ -446,8 +450,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 10000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 10000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, c, a0, b, c0"), term("joinType", "LeftOuterJoin") ), @@ -485,8 +489,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "proctime") ), term("where", - "AND(=(a, a0), >=(proctime, -(proctime0, 3600000)), " + - "<=(proctime, +(proctime0, 3600000)))"), + "AND(=(a, a0), >=(PROCTIME(proctime), -(PROCTIME(proctime0), 3600000)), " + + "<=(PROCTIME(proctime), +(PROCTIME(proctime0), 3600000)))"), term("join", "a, proctime, a0, b, proctime0"), term("joinType", "RightOuterJoin") ), @@ -523,8 +527,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 10000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 10000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, c, a0, b, c0"), term("joinType", "RightOuterJoin") ), @@ -562,8 +566,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "proctime") ), term("where", - "AND(=(a, a0), >=(proctime, -(proctime0, 3600000)), " + - "<=(proctime, +(proctime0, 3600000)))"), + "AND(=(a, a0), >=(PROCTIME(proctime), -(PROCTIME(proctime0), 3600000)), " + + "<=(PROCTIME(proctime), +(PROCTIME(proctime0), 3600000)))"), term("join", "a, proctime, a0, b, proctime0"), term("joinType", "FullOuterJoin") ), @@ -600,8 +604,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 10000)), " + - "<=(c, +(c0, 3600000)))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 10000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)))"), term("join", "a, c, a0, b, c0"), term("joinType", "FullOuterJoin") ), @@ -640,8 +644,8 @@ class JoinTest extends TableTestBase { term("select", "a", "b", "c") ), term("where", - "AND(=(a, a0), >=(c, -(c0, 10000)), " + - "<=(c, +(c0, 3600000)), LIKE(b, b0))"), + "AND(=(a, a0), >=(CAST(c), -(CAST(c0), 10000)), " + + "<=(CAST(c), +(CAST(c0), 3600000)), LIKE(b, b0))"), term("join", "a, b, c, a0, b0, c0"), // Since we filter on attributes b and b0 after the join, the full outer join // will be automatically optimized to inner join. @@ -786,33 +790,6 @@ class JoinTest extends TableTestBase { "AND(=($0, $4), >($2, $6))") } - private def verifyTimeBoundary( - timeSql: String, - expLeftSize: Long, - expRightSize: Long, - expTimeType: String): Unit = { - val query = - "SELECT t1.a, t2.b FROM MyTable as t1 join MyTable2 as t2 on t1.a = t2.a and " + timeSql - - val resultTable = streamUtil.tableEnv.sqlQuery(query) - val relNode = resultTable.getRelNode - val joinNode = relNode.getInput(0).asInstanceOf[LogicalJoin] - val (windowBounds, _) = - WindowJoinUtil.extractWindowBoundsFromPredicate( - joinNode.getCondition, - 4, - joinNode.getRowType, - joinNode.getCluster.getRexBuilder, - streamUtil.tableEnv.getConfig) - - val timeTypeStr = - if (windowBounds.get.isEventTime) "rowtime" - else "proctime" - assertEquals(expLeftSize, windowBounds.get.leftLowerBound) - assertEquals(expRightSize, windowBounds.get.leftUpperBound) - assertEquals(expTimeType, timeTypeStr) - } - @Test def testLeftOuterJoinEquiPred(): Unit = { val util = streamTestUtil() @@ -1003,12 +980,43 @@ class JoinTest extends TableTestBase { util.verifyTable(result, expected) } + private def verifyTimeBoundary( + timeSql: String, + expLeftSize: Long, + expRightSize: Long, + expTimeType: String): Unit = { + val query = + "SELECT t1.a, t2.b FROM MyTable as t1 join MyTable2 as t2 on t1.a = t2.a and " + timeSql + + val resultTable = streamUtil.tableEnv.sqlQuery(query) + val relNode = RelTimeIndicatorConverter.convert( + resultTable.getRelNode, + streamUtil.tableEnv.getRelBuilder.getRexBuilder) + val joinNode = relNode.getInput(0).asInstanceOf[LogicalJoin] + val (windowBounds, _) = + WindowJoinUtil.extractWindowBoundsFromPredicate( + joinNode.getCondition, + 4, + joinNode.getRowType, + joinNode.getCluster.getRexBuilder, + streamUtil.tableEnv.getConfig) + + val timeTypeStr = + if (windowBounds.get.isEventTime) "rowtime" + else "proctime" + assertEquals(expLeftSize, windowBounds.get.leftLowerBound) + assertEquals(expRightSize, windowBounds.get.leftUpperBound) + assertEquals(expTimeType, timeTypeStr) + } + private def verifyRemainConditionConvert( query: String, expectCondStr: String): Unit = { val resultTable = streamUtil.tableEnv.sqlQuery(query) - val relNode = resultTable.getRelNode + val relNode = RelTimeIndicatorConverter.convert( + resultTable.getRelNode, + streamUtil.tableEnv.getRelBuilder.getRexBuilder) val joinNode = relNode.getInput(0).asInstanceOf[LogicalJoin] val (_, remainCondition) = WindowJoinUtil.extractWindowBoundsFromPredicate( diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala index 3c47f562aae47..27c40bbbef24e 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala @@ -87,7 +87,9 @@ class TemporalTableJoinTest extends TableTestBase { val ratesHistory = util.addTable[(Timestamp, String, String, Int, Int)]( "RatesHistory", 'rowtime.rowtime, 'comment, 'currency, 'rate, 'secondary_key) - val rates = ratesHistory.createTemporalTableFunction('rowtime, 'currency) + val rates = ratesHistory + .filter('rate > 110L) + .createTemporalTableFunction('rowtime, 'currency) util.addFunction("Rates", rates) val sqlQuery = diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala index 141c8173ea049..2a994f31a2c64 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ import org.apache.flink.table.api.TableException +import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Test @@ -30,18 +30,6 @@ class JoinValidationTest extends TableTestBase { streamUtil.addTable[(Int, String, Long)]("MyTable", 'a, 'b, 'c.rowtime, 'proctime.proctime) streamUtil.addTable[(Int, String, Long)]("MyTable2", 'a, 'b, 'c.rowtime, 'proctime.proctime) - /** There should exist exactly two time conditions **/ - @Test(expected = classOf[TableException]) - def testWindowJoinSingleTimeCondition() = { - val sql = - """ - |SELECT t2.a - |FROM MyTable t1 JOIN MyTable2 t2 ON - | t1.a = t2.a AND - | t1.proctime > t2.proctime - INTERVAL '5' SECOND""".stripMargin - streamUtil.verifySql(sql, "n/a") - } - /** Both time attributes in a join condition must be of the same type **/ @Test(expected = classOf[TableException]) def testWindowJoinDiffTimeIndicator() = { @@ -136,4 +124,78 @@ class JoinValidationTest extends TableTestBase { streamUtil.verifySql(sql, "n/a") } + + /** Rowtime attributes cannot be accessed in filter conditions yet. */ + @Test + def testJoinWithRowtimeCondition(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage( + "Rowtime attributes must not be in the input rows of a regular join.") + + val sql = + """ + |SELECT t2.a + |FROM MyTable t1 JOIN MyTable2 t2 ON + | t1.a = t2.a AND + | t1.c > t2.c - INTERVAL '5' SECOND + |""".stripMargin + + streamUtil.verifySql(sql, "n/a") + } + + /** Rowtime attributes cannot be accessed in filter conditions yet. */ + @Test + def testJoinWithRowtimeConditionFromComplexQuery(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage( + "Rowtime attributes must not be in the input rows of a regular join.") + + val util = streamTestUtil() + + util.addTable[(Long, Long)]("MyTable1", 'id, 'eventTs.rowtime) + + util.addTable[(Long, Long)]("MyTable2", 'id, 'eventTs.rowtime) + + val sql1 = + """SELECT + | id, + | eventTs AS t1, + | COUNT(*) OVER ( + | PARTITION BY id ORDER BY eventTs ROWS BETWEEN 100 PRECEDING AND CURRENT ROW + | ) AS cnt1 + |FROM MyTable1 + |""".stripMargin + val sql2 = + """SELECT DISTINCT + | id AS r_id, + | eventTs AS t2, + | COUNT(*) OVER ( + | PARTITION BY id ORDER BY eventTs ROWS BETWEEN 50 PRECEDING AND CURRENT ROW + | ) AS cnt2 + |FROM MyTable2 + |""".stripMargin + + val left = util.tableEnv.sqlQuery(sql1) + val right = util.tableEnv.sqlQuery(sql2) + val result = left.join(right).where("id === r_id && t1 === t2").select("id, t1") + + util.verifyTable(result, "n/a") + } + + /** Rowtime attributes cannot be accessed in projection yet. */ + @Test + def testJoinWithRowtimeProjection(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage( + "Rowtime attributes must not be in the input rows of a regular join.") + + val sql = + """ + |SELECT t2.a, t2.c + |FROM MyTable t1 JOIN MyTable2 t2 ON + | t1.a = t2.a + |""".stripMargin + + streamUtil.verifySql(sql, "n/a") + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala index e7cf597ca64b0..5c237ffc5ee95 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala @@ -30,8 +30,11 @@ class WindowAggregateValidationTest extends TableTestBase { streamUtil.addTable[(Int, String, Long)]( "MyTable", 'a, 'b, 'c, 'proctime.proctime, 'rowtime.rowtime) - @Test(expected = classOf[TableException]) + @Test def testTumbleWindowNoOffset(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage("TUMBLE window with alignment is not supported yet") + val sqlQuery = "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " + "FROM MyTable " + @@ -40,8 +43,11 @@ class WindowAggregateValidationTest extends TableTestBase { streamUtil.verifySql(sqlQuery, "n/a") } - @Test(expected = classOf[TableException]) + @Test def testHopWindowNoOffset(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage("HOP window with alignment is not supported yet") + val sqlQuery = "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " + "FROM MyTable " + @@ -50,8 +56,11 @@ class WindowAggregateValidationTest extends TableTestBase { streamUtil.verifySql(sqlQuery, "n/a") } - @Test(expected = classOf[TableException]) + @Test def testSessionWindowNoOffset(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage("SESSION window with alignment is not supported yet") + val sqlQuery = "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " + "FROM MyTable " + @@ -60,20 +69,40 @@ class WindowAggregateValidationTest extends TableTestBase { streamUtil.verifySql(sqlQuery, "n/a") } - @Test(expected = classOf[TableException]) - def testVariableWindowSize() = { + @Test + def testVariableWindowSize(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage( + "Only constant window intervals with millisecond resolution are supported") + val sql = "SELECT COUNT(*) FROM MyTable GROUP BY TUMBLE(proctime, c * INTERVAL '1' MINUTE)" streamUtil.verifySql(sql, "n/a") } - @Test(expected = classOf[ValidationException]) + @Test def testWindowUdAggInvalidArgs(): Unit = { + expectedException.expect(classOf[ValidationException]) + expectedException.expectMessage("Given parameters of function do not match any signature") + streamUtil.tableEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge) val sqlQuery = "SELECT SUM(a) AS sumA, weightedAvg(a, b) AS wAvg " + "FROM MyTable " + - "GROUP BY TUMBLE(proctime(), INTERVAL '2' HOUR, TIME '10:00:00')" + "GROUP BY TUMBLE(proctime, INTERVAL '2' HOUR, TIME '10:00:00')" + + streamUtil.verifySql(sqlQuery, "n/a") + } + + @Test + def testWindowWrongWindowParameter(): Unit = { + expectedException.expect(classOf[TableException]) + expectedException.expectMessage( + "Only constant window intervals with millisecond resolution are supported") + + val sqlQuery = + "SELECT COUNT(*) FROM MyTable " + + "GROUP BY TUMBLE(proctime, INTERVAL '2-10' YEAR TO MONTH)" streamUtil.verifySql(sqlQuery, "n/a") } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala index 671f8dd8d1dab..afa9f8b0c7947 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala @@ -21,6 +21,7 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.scala._ +import org.apache.flink.table.plan.logical.{SessionGroupWindow, SlidingGroupWindow, TumblingGroupWindow} import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.TableTestBase @@ -238,4 +239,78 @@ class AggregateTest extends TableTestBase { util.verifyTable(resultTable, expected) } + + @Test + def testDistinctAggregateOnTumbleWindow(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Long, String)]( + "MyTable", 'a, 'b, 'c, 'rowtime.rowtime) + val result = table + .window(Tumble over 15.minute on 'rowtime as 'w) + .groupBy('w) + .select('a.count.distinct, 'a.sum) + + val expected = unaryNode( + "DataStreamGroupWindowAggregate", + unaryNode( + "DataStreamCalc", + streamTableNode(0), + term("select", "a", "rowtime") + ), + term("window", TumblingGroupWindow('w, 'rowtime, 900000.millis)), + term("select", "COUNT(DISTINCT a) AS TMP_0", "SUM(a) AS TMP_1") + ) + + util.verifyTable(result, expected) + } + + @Test + def testMultiDistinctAggregateSameFieldOnHopWindow(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Long, String)]( + "MyTable", 'a, 'b, 'c, 'rowtime.rowtime) + val result = table + .window(Slide over 1.hour every 15.minute on 'rowtime as 'w) + .groupBy('w) + .select('a.count.distinct, 'a.sum.distinct, 'a.max.distinct) + + val expected = unaryNode( + "DataStreamGroupWindowAggregate", + unaryNode( + "DataStreamCalc", + streamTableNode(0), + term("select", "a", "rowtime") + ), + term("window", SlidingGroupWindow('w, 'rowtime, 3600000.millis, 900000.millis)), + term("select", "COUNT(DISTINCT a) AS TMP_0", "SUM(DISTINCT a) AS TMP_1", + "MAX(DISTINCT a) AS TMP_2") + ) + + util.verifyTable(result, expected) + } + + @Test + def testDistinctAggregateWithGroupingOnSessionWindow(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Long, String)]( + "MyTable", 'a, 'b, 'c, 'rowtime.rowtime) + val result = table + .window(Session withGap 15.minute on 'rowtime as 'w) + .groupBy('a, 'w) + .select('a, 'a.count, 'c.count.distinct) + + val expected = unaryNode( + "DataStreamGroupWindowAggregate", + unaryNode( + "DataStreamCalc", + streamTableNode(0), + term("select", "a", "c", "rowtime") + ), + term("groupBy", "a"), + term("window", SessionGroupWindow('w, 'rowtime, 900000.millis)), + term("select", "a", "COUNT(a) AS TMP_0", "COUNT(DISTINCT c) AS TMP_1") + ) + + util.verifyTable(result, expected) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala index d7f5c71581592..138497cada3dd 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala @@ -27,7 +27,7 @@ import org.apache.flink.table.utils.TableTestUtil._ import org.junit.Test /** - * Currently only time-windowed joins can be processed in a streaming fashion. + * Tests for both windowed and non-windowed joins. */ class JoinTest extends TableTestBase { @@ -57,8 +57,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rrtime") ), - term("where", "AND(=(a, d), >=(lrtime, -(rrtime, 300000))," + - " <(lrtime, +(rrtime, 3000)))"), + term("where", "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000))," + + " <(CAST(lrtime), +(CAST(rrtime), 3000)))"), term("join", "a", "lrtime", "d", "e", "rrtime"), term("joinType", "InnerJoin") ), @@ -92,7 +92,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rptime") ), - term("where", "AND(=(a, d), >=(lptime, -(rptime, 1000)), <(lptime, rptime))"), + term("where", "AND(=(a, d), >=(PROCTIME(lptime), -(PROCTIME(rptime), 1000)), " + + "<(PROCTIME(lptime), PROCTIME(rptime)))"), term("join", "a", "lptime", "d", "e", "rptime"), term("joinType", "InnerJoin") ), @@ -126,7 +127,7 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rptime") ), - term("where", "AND(=(a, d), =(lptime, rptime))"), + term("where", "AND(=(a, d), =(PROCTIME(lptime), PROCTIME(rptime)))"), term("join", "a", "lptime", "d", "e", "rptime"), term("joinType", "InnerJoin") ), @@ -153,7 +154,8 @@ class JoinTest extends TableTestBase { streamTableNode(0), streamTableNode(1), term("where", - "AND(=(a, d), >=(lrtime, -(rrtime, 300000)), <(lrtime, rrtime), >(lrtime, " + "f))"), + "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000)), " + + "<(CAST(lrtime), CAST(rrtime)), >(CAST(lrtime), f))"), term("join", "a", "b", "c", "lrtime", "d", "e", "f", "rrtime"), term("joinType", "InnerJoin") ) @@ -188,8 +190,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rrtime") ), - term("where", "AND(=(a, d), >=(lrtime, -(rrtime, 300000))," + - " <(lrtime, +(rrtime, 3000)))"), + term("where", "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000))," + + " <(CAST(lrtime), +(CAST(rrtime), 3000)))"), term("join", "a", "lrtime", "d", "e", "rrtime"), term("joinType", "LeftOuterJoin") ), @@ -223,7 +225,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rptime") ), - term("where", "AND(=(a, d), >=(lptime, -(rptime, 1000)), <(lptime, rptime))"), + term("where", "AND(=(a, d), >=(PROCTIME(lptime), -(PROCTIME(rptime), 1000)), " + + "<(PROCTIME(lptime), PROCTIME(rptime)))"), term("join", "a", "lptime", "d", "e", "rptime"), term("joinType", "LeftOuterJoin") ), @@ -260,8 +263,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rrtime") ), - term("where", "AND(=(a, d), >=(lrtime, -(rrtime, 300000))," + - " <(lrtime, +(rrtime, 3000)))"), + term("where", "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000))," + + " <(CAST(lrtime), +(CAST(rrtime), 3000)))"), term("join", "a", "lrtime", "d", "e", "rrtime"), term("joinType", "RightOuterJoin") ), @@ -295,7 +298,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rptime") ), - term("where", "AND(=(a, d), >=(lptime, -(rptime, 1000)), <(lptime, rptime))"), + term("where", "AND(=(a, d), >=(PROCTIME(lptime), -(PROCTIME(rptime), 1000)), " + + "<(PROCTIME(lptime), PROCTIME(rptime)))"), term("join", "a", "lptime", "d", "e", "rptime"), term("joinType", "RightOuterJoin") ), @@ -332,8 +336,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rrtime") ), - term("where", "AND(=(a, d), >=(lrtime, -(rrtime, 300000))," + - " <(lrtime, +(rrtime, 3000)))"), + term("where", "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000))," + + " <(CAST(lrtime), +(CAST(rrtime), 3000)))"), term("join", "a", "lrtime", "d", "e", "rrtime"), term("joinType", "FullOuterJoin") ), @@ -367,7 +371,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rptime") ), - term("where", "AND(=(a, d), >=(lptime, -(rptime, 1000)), <(lptime, rptime))"), + term("where", "AND(=(a, d), >=(PROCTIME(lptime), -(PROCTIME(rptime), 1000)), " + + "<(PROCTIME(lptime), PROCTIME(rptime)))"), term("join", "a", "lptime", "d", "e", "rptime"), term("joinType", "FullOuterJoin") ), @@ -402,8 +407,8 @@ class JoinTest extends TableTestBase { streamTableNode(1), term("select", "d", "e", "rrtime") ), - term("where", "AND(=(a, d), >=(lrtime, -(rrtime, 300000))," + - " <(lrtime, +(rrtime, 3000)))"), + term("where", "AND(=(a, d), >=(CAST(lrtime), -(CAST(rrtime), 300000))," + + " <(CAST(lrtime), +(CAST(rrtime), 3000)))"), term("join", "a", "lrtime", "d", "e", "rrtime"), // Since we filter on attributes of the left table after the join, the left outer join // will be automatically optimized to inner join. diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala index f8d49238430e7..299c14417b62d 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala @@ -87,7 +87,9 @@ class TemporalTableJoinTest extends TableTestBase { val ratesHistory = util.addTable[(Timestamp, String, String, Int, Int)]( "RatesHistory", 'rowtime.rowtime, 'comment, 'currency, 'rate, 'secondary_key) - val rates = ratesHistory.createTemporalTableFunction('rowtime, 'currency) + val rates = ratesHistory + .filter('rate > 110L) + .createTemporalTableFunction('rowtime, 'currency) util.addFunction("Rates", rates) val result = orders @@ -226,7 +228,8 @@ object TemporalTableJoinTest { unaryNode( "DataStreamCalc", streamTableNode(2), - term("select", "rowtime, currency, rate, secondary_key") + term("select", "rowtime, currency, rate, secondary_key"), + term("where", ">(rate, 110)") ), term("where", "AND(" + diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala index b2a8ec9f567d8..fe7c75df6cf78 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala @@ -20,6 +20,7 @@ package org.apache.flink.table.descriptors import java.util import java.util.Collections +import java.util.function.Consumer import org.apache.flink.table.api.ValidationException import org.apache.flink.table.util.JavaScalaConversionUtil.toJava @@ -32,6 +33,9 @@ import org.junit.Test class DescriptorPropertiesTest { private val ARRAY_KEY = "my-array" + private val FIXED_INDEXED_PROPERTY_KEY = "my-fixed-indexed-property" + private val PROPERTY_1_KEY = "property-1" + private val PROPERTY_2_KEY = "property-2" @Test def testEquals(): Unit = { @@ -97,8 +101,8 @@ class DescriptorPropertiesTest { def testArrayInvalidValues(): Unit = { val properties = new DescriptorProperties() properties.putString(s"$ARRAY_KEY.0", "12") - properties.putString(s"$ARRAY_KEY.1", "INVALID") - properties.putString(s"$ARRAY_KEY.2", "66") + properties.putString(s"$ARRAY_KEY.1", "66") + properties.putString(s"$ARRAY_KEY.2", "INVALID") testArrayValidation(properties, 1, Integer.MAX_VALUE) } @@ -118,6 +122,19 @@ class DescriptorPropertiesTest { testArrayValidation(properties, 1, Integer.MAX_VALUE) } + @Test(expected = classOf[ValidationException]) + def testInvalidFixedIndexedProperties(): Unit = { + val property = new DescriptorProperties() + val list = new util.ArrayList[util.List[String]]() + list.add(util.Arrays.asList("1", "string")) + list.add(util.Arrays.asList("INVALID", "string")) + property.putIndexedFixedProperties( + FIXED_INDEXED_PROPERTY_KEY, + util.Arrays.asList(PROPERTY_1_KEY, PROPERTY_2_KEY), + list) + testFixedIndexedPropertiesValidation(property) + } + @Test def testRemoveKeys(): Unit = { val properties = new DescriptorProperties() @@ -155,7 +172,7 @@ class DescriptorPropertiesTest { minLength: Int, maxLength: Int) : Unit = { - val validator: (String) => Unit = (key: String) => { + val validator: String => Unit = (key: String) => { properties.validateInt(key, false) } @@ -165,4 +182,26 @@ class DescriptorPropertiesTest { minLength, maxLength) } + + private def testFixedIndexedPropertiesValidation(properties: DescriptorProperties): Unit = { + + val validatorMap = new util.HashMap[String, Consumer[String]]() + + // PROPERTY_1 should be Int + val validator1: String => Unit = (key: String) => { + properties.validateInt(key, false) + } + validatorMap.put(PROPERTY_1_KEY, toJava(validator1)) + // PROPERTY_2 should be String + val validator2: String => Unit = (key: String) => { + properties.validateString(key, false) + } + validatorMap.put(PROPERTY_2_KEY, toJava(validator2)) + + properties.validateFixedIndexedProperties( + FIXED_INDEXED_PROPERTY_KEY, + false, + validatorMap + ) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala index 42513c2a96fc1..23bd2ae337998 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala @@ -645,6 +645,13 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { "'foobar'.regexpReplace('oo|ar', f33)", "REGEXP_REPLACE('foobar', 'oo|ar', f33)", "null") + + // This test was added for the null literal problem in string expression parsing (FLINK-10463). + testAllApis( + Null(Types.STRING).regexpReplace("oo|ar", 'f33), + "Null(STRING).regexpReplace('oo|ar', f33)", + "REGEXP_REPLACE(CAST(NULL AS VARCHAR), 'oo|ar', f33)", + "null") } @Test @@ -2695,17 +2702,73 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { "TIMESTAMPADD(HOUR, -200, CAST(NULL AS TIMESTAMP))", "null") - testAllApis( - "2016-06-15".toDate + 1.day, - "'2016-06-15'.toDate + 1.day", - "TIMESTAMPADD(DAY, 1, DATE '2016-06-15')", - "2016-06-16") - testAllApis( Null(Types.SQL_TIMESTAMP) + 3.months, "Null(SQL_TIMESTAMP) + 3.months", "TIMESTAMPADD(MONTH, 3, CAST(NULL AS TIMESTAMP))", "null") + + // TIMESTAMPADD with DATE returns a TIMESTAMP value for sub-day intervals. + testAllApis("2016-06-15".toDate + 1.month, + "'2016-06-15'.toDate + 1.month", + "timestampadd(MONTH, 1, date '2016-06-15')", + "2016-07-15") + + testAllApis("2016-06-15".toDate + 1.day, + "'2016-06-15'.toDate + 1.day", + "timestampadd(DAY, 1, date '2016-06-15')", + "2016-06-16") + + testAllApis("2016-06-15".toTimestamp - 1.hour, + "'2016-06-15'.toTimestamp - 1.hour", + "timestampadd(HOUR, -1, date '2016-06-15')", + "2016-06-14 23:00:00.0") + + testAllApis("2016-06-15".toTimestamp + 1.minute, + "'2016-06-15'.toTimestamp + 1.minute", + "timestampadd(MINUTE, 1, date '2016-06-15')", + "2016-06-15 00:01:00.0") + + testAllApis("2016-06-15".toTimestamp - 1.second, + "'2016-06-15'.toTimestamp - 1.second", + "timestampadd(SQL_TSI_SECOND, -1, date '2016-06-15')", + "2016-06-14 23:59:59.0") + + testAllApis("2016-06-15".toTimestamp + 1.second, + "'2016-06-15'.toTimestamp + 1.second", + "timestampadd(SECOND, 1, date '2016-06-15')", + "2016-06-15 00:00:01.0") + + testAllApis(Null(Types.SQL_TIMESTAMP) + 1.second, + "Null(SQL_TIMESTAMP) + 1.second", + "timestampadd(SECOND, 1, cast(null as date))", + "null") + + testAllApis(Null(Types.SQL_TIMESTAMP) + 1.day, + "Null(SQL_TIMESTAMP) + 1.day", + "timestampadd(DAY, 1, cast(null as date))", + "null") + + // Round to the last day of previous month + testAllApis("2016-05-31".toDate + 1.month, + "'2016-05-31'.toDate + 1.month", + "timestampadd(MONTH, 1, date '2016-05-31')", + "2016-06-30") + + testAllApis("2016-01-31".toDate + 5.month, + "'2016-01-31'.toDate + 5.month", + "timestampadd(MONTH, 5, date '2016-01-31')", + "2016-06-30") + + testAllApis("2016-03-31".toDate - 1.month, + "'2016-03-31'.toDate - 1.month", + "timestampadd(MONTH, -1, date '2016-03-31')", + "2016-02-29") + + testAllApis("2016-03-31".toDate - 1.week, + "'2016-03-31'.toDate - 1.week", + "timestampadd(WEEK, -1, date '2016-03-31')", + "2016-03-24") } // ---------------------------------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala index b8adfa78efbc7..f9220e68f0f3f 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala @@ -67,8 +67,7 @@ abstract class ExpressionTestBase { private val planner = new FlinkPlannerImpl( context._2.getFrameworkConfig, context._2.getPlanner, - context._2.getTypeFactory, - context._2.sqlToRelConverterConfig) + context._2.getTypeFactory) private val logicalOptProgram = Programs.ofRules(FlinkRuleSets.LOGICAL_OPT_RULES) private val dataSetOptProgram = Programs.ofRules(FlinkRuleSets.DATASET_OPT_RULES) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/userDefinedScalarFunctions.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/userDefinedScalarFunctions.scala index 6d02afca510a9..912bb047cba0b 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/userDefinedScalarFunctions.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/userDefinedScalarFunctions.scala @@ -290,6 +290,9 @@ object Func19 extends ScalarFunction { } +/** + * A scalar function that always returns TRUE if opened correctly. + */ class Func20 extends ScalarFunction { private var permitted: Boolean = false diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala index 3917bdf8af934..4fa8b175a0e72 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala @@ -179,6 +179,35 @@ class MatchOperatorValidationTest extends TableTestBase { streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row] } + @Test + def testValidatingAmbiguousColumns(): Unit = { + thrown.expectMessage("Columns ambiguously defined: {symbol, price}") + thrown.expect(classOf[ValidationException]) + + val sqlQuery = + s""" + |SELECT * + |FROM Ticker + |MATCH_RECOGNIZE ( + | PARTITION BY symbol, price + | ORDER BY proctime + | MEASURES + | A.symbol AS symbol, + | A.price AS price + | PATTERN (A) + | DEFINE + | A AS symbol = 'a' + |) AS T + |""".stripMargin + + streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row] + } + + // *************************************************************************************** + // * Those validations are temporary. We should remove those tests once we support those * + // * features. * + // *************************************************************************************** + @Test def testPatternsProducingEmptyMatchesAreNotSupported(): Unit = { thrown.expectMessage("Patterns that can produce empty matches are not supported. " + diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala index 55bdb5245c2be..883ce0adf8102 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala @@ -49,8 +49,7 @@ abstract class PatternTranslatorTestBase extends TestLogger{ private val planner = new FlinkPlannerImpl( context._2.getFrameworkConfig, context._2.getPlanner, - context._2.getTypeFactory, - context._2.sqlToRelConverterConfig) + context._2.getTypeFactory) private def prepareContext(typeInfo: TypeInformation[Row]) : (RelBuilder, StreamTableEnvironment, StreamExecutionEnvironment) = { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala index 1706169215398..29dda21f93226 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala @@ -85,7 +85,7 @@ class TimeIndicatorConversionTest extends TableTestBase { "DataStreamCalc", streamTableNode(0), term("select", "rowtime"), - term("where", ">(rowtime, 1990-12-02 12:11:11)") + term("where", ">(CAST(rowtime), 1990-12-02 12:11:11)") ) util.verifyTable(result, expected) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala index b161eeda43cd0..59f214bcd1ea9 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala @@ -578,6 +578,22 @@ class CalcITCase( val expected = List("a,a,d,d,e,e", "x,x,z,z,z,z").mkString("\n") TestBaseUtils.compareResultAsText(results.asJava, expected) } + + @Test + def testSplitFieldsOnCustomType(): Unit = { + val env = ExecutionEnvironment.getExecutionEnvironment + val tEnv = TableEnvironment.getTableEnvironment(env, config) + tEnv.getConfig.setMaxGeneratedCodeLength(1) // splits fields + + val ds = CollectionDataSets.getCustomTypeDataSet(env) + val filterDs = ds.toTable(tEnv, 'myInt as 'i, 'myLong as 'l, 'myString as 's) + .filter('s.like("%a%") && 's.charLength() > 12) + .select('i, 'l, 's.charLength()) + + val expected = "3,3,25\n" + "3,5,14\n" + val results = filterDs.toDataSet[Row].collect() + TestBaseUtils.compareResultAsText(results.asJava, expected) + } } object CalcITCase { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala index b7d349c29fcd4..2dbe1cddf00cd 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala @@ -26,6 +26,7 @@ import org.apache.flink.api.scala.util.CollectionDataSets import org.apache.flink.table.api.TableEnvironment import org.apache.flink.table.api.scala._ import org.apache.flink.table.expressions.Literal +import org.apache.flink.table.expressions.utils.Func20 import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.utils.TableFunc2 @@ -53,7 +54,12 @@ class JoinITCase( val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c) val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h) - val joinT = ds1.join(ds2).where('b === 'e).select('c, 'g) + val testOpenCall = new Func20 + + val joinT = ds1.join(ds2) + .where('b === 'e) + .where(testOpenCall('a + 'd)) + .select('c, 'g) val expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n" val results = joinT.toDataSet[Row].collect() diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala similarity index 65% rename from flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala rename to flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala index 7c4f54303289a..1dce9946877ff 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala @@ -24,20 +24,18 @@ import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.StreamQueryConfig import org.apache.flink.table.runtime.aggregate._ import org.apache.flink.table.runtime.harness.HarnessTestBase._ import org.apache.flink.table.runtime.types.CRow -import org.apache.flink.types.Row import org.junit.Test -class NonWindowHarnessTest extends HarnessTestBase { +class GroupAggregateHarnessTest extends HarnessTestBase { protected var queryConfig = new TestStreamQueryConfig(Time.seconds(2), Time.seconds(3)) @Test - def testNonWindow(): Unit = { + def testAggregate(): Unit = { val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new GroupAggProcessFunction( @@ -54,50 +52,49 @@ class NonWindowHarnessTest extends HarnessTestBase { testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + // register cleanup timer with 3001 testHarness.setProcessingTime(1) testHarness.processElement(new StreamRecord(CRow(1L: JLong, 1: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(2L: JLong, 1: JInt, "bbb"), 1)) + expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1: JInt), 1)) // reuse timer 3001 testHarness.setProcessingTime(1000) testHarness.processElement(new StreamRecord(CRow(3L: JLong, 2: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(3L: JLong, 3: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(4L: JLong, 3: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(4L: JLong, 6: JInt), 1)) // register cleanup timer with 4002 testHarness.setProcessingTime(1002) testHarness.processElement(new StreamRecord(CRow(5L: JLong, 4: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(5L: JLong, 10: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(6L: JLong, 2: JInt, "bbb"), 1)) + expectedOutput.add(new StreamRecord(CRow(6L: JLong, 3: JInt), 1)) // trigger cleanup timer and register cleanup timer with 7003 testHarness.setProcessingTime(4003) testHarness.processElement(new StreamRecord(CRow(7L: JLong, 5: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(7L: JLong, 5: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(8L: JLong, 6: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(8L: JLong, 11: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(9L: JLong, 7: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(9L: JLong, 18: JInt), 1)) testHarness.processElement(new StreamRecord(CRow(10L: JLong, 3: JInt, "bbb"), 1)) + expectedOutput.add(new StreamRecord(CRow(10L: JLong, 3: JInt), 1)) val result = testHarness.getOutput - val expectedOutput = new ConcurrentLinkedQueue[Object]() - - expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(3L: JLong, 3: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(4L: JLong, 6: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(5L: JLong, 10: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(6L: JLong, 3: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(7L: JLong, 5: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(8L: JLong, 11: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(9L: JLong, 18: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(10L: JLong, 3: JInt), 1)) - verify(expectedOutput, result) testHarness.close() } @Test - def testNonWindowWithRetract(): Unit = { + def testAggregateWithRetract(): Unit = { val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new GroupAggProcessFunction( @@ -114,42 +111,136 @@ class NonWindowHarnessTest extends HarnessTestBase { testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + // register cleanup timer with 3001 testHarness.setProcessingTime(1) + // accumulate testHarness.processElement(new StreamRecord(CRow(1L: JLong, 1: JInt, "aaa"), 1)) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1: JInt), 1)) + + // accumulate testHarness.processElement(new StreamRecord(CRow(2L: JLong, 1: JInt, "bbb"), 2)) + expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1: JInt), 2)) + + // retract for insertion testHarness.processElement(new StreamRecord(CRow(3L: JLong, 2: JInt, "aaa"), 3)) + expectedOutput.add(new StreamRecord(CRow(false, 3L: JLong, 1: JInt), 3)) + expectedOutput.add(new StreamRecord(CRow(3L: JLong, 3: JInt), 3)) + + // retract for deletion + testHarness.processElement(new StreamRecord(CRow(false, 3L: JLong, 2: JInt, "aaa"), 3)) + expectedOutput.add(new StreamRecord(CRow(false, 3L: JLong, 3: JInt), 3)) + expectedOutput.add(new StreamRecord(CRow(3L: JLong, 1: JInt), 3)) + + // accumulate testHarness.processElement(new StreamRecord(CRow(4L: JLong, 3: JInt, "ccc"), 4)) + expectedOutput.add(new StreamRecord(CRow(4L: JLong, 3: JInt), 4)) // trigger cleanup timer and register cleanup timer with 6002 testHarness.setProcessingTime(3002) - testHarness.processElement(new StreamRecord(CRow(5L: JLong, 4: JInt, "aaa"), 5)) - testHarness.processElement(new StreamRecord(CRow(6L: JLong, 2: JInt, "bbb"), 6)) - testHarness.processElement(new StreamRecord(CRow(7L: JLong, 5: JInt, "aaa"), 7)) - testHarness.processElement(new StreamRecord(CRow(8L: JLong, 6: JInt, "eee"), 8)) - testHarness.processElement(new StreamRecord(CRow(9L: JLong, 7: JInt, "aaa"), 9)) - testHarness.processElement(new StreamRecord(CRow(10L: JLong, 3: JInt, "bbb"), 10)) - - val result = testHarness.getOutput - val expectedOutput = new ConcurrentLinkedQueue[Object]() + // retract after clean up + testHarness.processElement(new StreamRecord(CRow(false, 4L: JLong, 3: JInt, "ccc"), 4)) - expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1: JInt), 1)) - expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1: JInt), 2)) - expectedOutput.add(new StreamRecord(CRow(false, 3L: JLong, 1: JInt), 3)) - expectedOutput.add(new StreamRecord(CRow(3L: JLong, 3: JInt), 3)) - expectedOutput.add(new StreamRecord(CRow(4L: JLong, 3: JInt), 4)) + // accumulate + testHarness.processElement(new StreamRecord(CRow(5L: JLong, 4: JInt, "aaa"), 5)) expectedOutput.add(new StreamRecord(CRow(5L: JLong, 4: JInt), 5)) + testHarness.processElement(new StreamRecord(CRow(6L: JLong, 2: JInt, "bbb"), 6)) expectedOutput.add(new StreamRecord(CRow(6L: JLong, 2: JInt), 6)) + + // retract + testHarness.processElement(new StreamRecord(CRow(7L: JLong, 5: JInt, "aaa"), 7)) expectedOutput.add(new StreamRecord(CRow(false, 7L: JLong, 4: JInt), 7)) expectedOutput.add(new StreamRecord(CRow(7L: JLong, 9: JInt), 7)) + + // accumulate + testHarness.processElement(new StreamRecord(CRow(8L: JLong, 6: JInt, "eee"), 8)) expectedOutput.add(new StreamRecord(CRow(8L: JLong, 6: JInt), 8)) + + // retract + testHarness.processElement(new StreamRecord(CRow(9L: JLong, 7: JInt, "aaa"), 9)) expectedOutput.add(new StreamRecord(CRow(false, 9L: JLong, 9: JInt), 9)) expectedOutput.add(new StreamRecord(CRow(9L: JLong, 16: JInt), 9)) + testHarness.processElement(new StreamRecord(CRow(10L: JLong, 3: JInt, "bbb"), 10)) expectedOutput.add(new StreamRecord(CRow(false, 10L: JLong, 2: JInt), 10)) expectedOutput.add(new StreamRecord(CRow(10L: JLong, 5: JInt), 10)) + val result = testHarness.getOutput + + verify(expectedOutput, result) + + testHarness.close() + } + + @Test + def testDistinctAggregateWithRetract(): Unit = { + + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( + new GroupAggProcessFunction( + genDistinctCountAggFunction, + distinctCountAggregationStateType, + true, + queryConfig)) + + val testHarness = + createHarnessTester( + processFunction, + new TupleRowKeySelector[String](2), + BasicTypeInfo.STRING_TYPE_INFO) + + testHarness.open() + + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + // register cleanup timer with 3001 + testHarness.setProcessingTime(1) + + // insert + testHarness.processElement(new StreamRecord(CRow(1L: JLong, 1: JInt, "aaa"))) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1L: JLong))) + testHarness.processElement(new StreamRecord(CRow(2L: JLong, 1: JInt, "bbb"))) + expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1L: JLong))) + + // distinct count retract then accumulate for downstream operators + testHarness.processElement(new StreamRecord(CRow(2L: JLong, 1: JInt, "bbb"))) + expectedOutput.add(new StreamRecord(CRow(false, 2L: JLong, 1L: JLong))) + expectedOutput.add(new StreamRecord(CRow(2L: JLong, 1L: JLong))) + + // update count for accumulate + testHarness.processElement(new StreamRecord(CRow(1L: JLong, 2: JInt, "aaa"))) + expectedOutput.add(new StreamRecord(CRow(false, 1L: JLong, 1L: JLong))) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 2L: JLong))) + + // update count for retraction + testHarness.processElement(new StreamRecord(CRow(false, 1L: JLong, 2: JInt, "aaa"))) + expectedOutput.add(new StreamRecord(CRow(false, 1L: JLong, 2L: JLong))) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1L: JLong))) + + // insert + testHarness.processElement(new StreamRecord(CRow(4L: JLong, 3: JInt, "ccc"))) + expectedOutput.add(new StreamRecord(CRow(4L: JLong, 1L: JLong))) + + // retract entirely + testHarness.processElement(new StreamRecord(CRow(false, 4L: JLong, 3: JInt, "ccc"))) + expectedOutput.add(new StreamRecord(CRow(false, 4L: JLong, 1L: JLong))) + + // trigger cleanup timer and register cleanup timer with 6002 + testHarness.setProcessingTime(3002) + + // insert + testHarness.processElement(new StreamRecord(CRow(1L: JLong, 1: JInt, "aaa"))) + expectedOutput.add(new StreamRecord(CRow(1L: JLong, 1L: JLong))) + + // trigger cleanup timer and register cleanup timer with 9002 + testHarness.setProcessingTime(6002) + + // retract after cleanup + testHarness.processElement(new StreamRecord(CRow(false, 1L: JLong, 1: JInt, "aaa"))) + + val result = testHarness.getOutput + verify(expectedOutput, result) testHarness.close() diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala index f70d991e50bd0..e5cceecc560b5 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala @@ -19,8 +19,9 @@ package org.apache.flink.table.runtime.harness import java.util.{Comparator, Queue => JQueue} +import org.apache.flink.api.common.state.{MapStateDescriptor, StateDescriptor} import org.apache.flink.api.common.time.Time -import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{INT_TYPE_INFO, LONG_TYPE_INFO, STRING_TYPE_INFO} +import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{LONG_TYPE_INFO, STRING_TYPE_INFO} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.functions.KeySelector import org.apache.flink.api.java.typeutils.RowTypeInfo @@ -28,11 +29,11 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil} -import org.apache.flink.table.api.StreamQueryConfig +import org.apache.flink.table.api.{StreamQueryConfig, Types} import org.apache.flink.table.codegen.GeneratedAggregationsFunction -import org.apache.flink.table.functions.{AggregateFunction, UserDefinedFunction} -import org.apache.flink.table.functions.aggfunctions.{IntSumWithRetractAggFunction, LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction} +import org.apache.flink.table.functions.aggfunctions.{CountAggFunction, IntSumWithRetractAggFunction, LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction} import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.getAccumulatorTypeOfAggregateFunction +import org.apache.flink.table.functions.{AggregateFunction, UserDefinedFunction} import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, RowResultSortComparatorWithWatermarks} import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo} import org.apache.flink.table.utils.EncodingUtils @@ -55,20 +56,16 @@ class HarnessTestBase { val intSumWithRetractAggFunction: String = EncodingUtils.encodeObjectToString(new IntSumWithRetractAggFunction) + val distinctCountAggFunction: String = + EncodingUtils.encodeObjectToString(new CountAggFunction()) + protected val MinMaxRowType = new RowTypeInfo(Array[TypeInformation[_]]( LONG_TYPE_INFO, STRING_TYPE_INFO, LONG_TYPE_INFO), Array("rowtime", "a", "b")) - protected val SumRowType = new RowTypeInfo(Array[TypeInformation[_]]( - LONG_TYPE_INFO, - INT_TYPE_INFO, - STRING_TYPE_INFO), - Array("a", "b", "c")) - protected val minMaxCRowType = new CRowTypeInfo(MinMaxRowType) - protected val sumCRowType = new CRowTypeInfo(SumRowType) protected val minMaxAggregates: Array[AggregateFunction[_, _]] = Array(new LongMinWithRetractAggFunction, @@ -77,15 +74,28 @@ class HarnessTestBase { protected val sumAggregates: Array[AggregateFunction[_, _]] = Array(new IntSumWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]] + protected val distinctCountAggregates: Array[AggregateFunction[_, _]] = + Array(new CountAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]] + protected val minMaxAggregationStateType: RowTypeInfo = new RowTypeInfo(minMaxAggregates.map(getAccumulatorTypeOfAggregateFunction(_)): _*) protected val sumAggregationStateType: RowTypeInfo = new RowTypeInfo(sumAggregates.map(getAccumulatorTypeOfAggregateFunction(_)): _*) + protected val distinctCountAggregationStateType: RowTypeInfo = + new RowTypeInfo(distinctCountAggregates.map(getAccumulatorTypeOfAggregateFunction(_)): _*) + + protected val distinctCountDescriptor: String = EncodingUtils.encodeObjectToString( + new MapStateDescriptor("distinctAgg0", distinctCountAggregationStateType, Types.LONG)) + + protected val minMaxFuncName = "MinMaxAggregateHelper" + protected val sumFuncName = "SumAggregationHelper" + protected val distinctCountFuncName = "DistinctCountAggregationHelper" + val minMaxCode: String = s""" - |public class MinMaxAggregateHelper + |public class $minMaxFuncName | extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations { | | transient org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction @@ -94,7 +104,7 @@ class HarnessTestBase { | transient org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction | fmax = null; | - | public MinMaxAggregateHelper() throws Exception { + | public $minMaxFuncName() throws Exception { | | fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction) | ${classOf[EncodingUtils].getCanonicalName}.decodeStringToObject( @@ -207,25 +217,25 @@ class HarnessTestBase { val sumAggCode: String = s""" - |public final class SumAggregationHelper + |public final class $sumFuncName | extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations { | | - |transient org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction - |sum = null; - |private final org.apache.flink.table.runtime.aggregate.SingleElementIterable accIt0 = | new org.apache.flink.table.runtime.aggregate.SingleElementIterable(); | - | public SumAggregationHelper() throws Exception { + | public $sumFuncName() throws Exception { | - |sum = (org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction) - |${classOf[EncodingUtils].getCanonicalName}.decodeStringToObject( - | "$intSumWithRetractAggFunction", - | ${classOf[UserDefinedFunction].getCanonicalName}.class); - |} + | sum = (org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction) + | ${classOf[EncodingUtils].getCanonicalName}.decodeStringToObject( + | "$intSumWithRetractAggFunction", + | ${classOf[UserDefinedFunction].getCanonicalName}.class); + | } | | public final void setAggregationResults( | org.apache.flink.types.Row accs, @@ -256,6 +266,12 @@ class HarnessTestBase { | public final void retract( | org.apache.flink.types.Row accs, | org.apache.flink.types.Row input) { + | + | sum.retract( + | ((org.apache.flink.table.functions.aggfunctions.SumWithRetractAccumulator) accs + | .getField + | (0)), + | (java.lang.Integer) input.getField(1)); | } | | public final org.apache.flink.types.Row createAccumulators() @@ -281,6 +297,162 @@ class HarnessTestBase { | input.getField(0)); | } | + | public final org.apache.flink.types.Row createOutputRow() { + | return new org.apache.flink.types.Row(2); + | } + | + | + | public final org.apache.flink.types.Row mergeAccumulatorsPair( + | org.apache.flink.types.Row a, + | org.apache.flink.types.Row b) + | { + | + | return a; + | + | } + | + | public final void resetAccumulator( + | org.apache.flink.types.Row accs) { + | } + | + | public void open(org.apache.flink.api.common.functions.RuntimeContext ctx) { + | } + | + | public void cleanup() { + | } + | + | public void close() { + | } + |} + |""".stripMargin + + val distinctCountAggCode: String = + s""" + |public final class $distinctCountFuncName + | extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations { + | + | final org.apache.flink.table.functions.aggfunctions.CountAggFunction count; + | + | final org.apache.flink.table.api.dataview.MapView acc0_distinctValueMap_dataview; + | + | final java.lang.reflect.Field distinctValueMap = + | org.apache.flink.api.java.typeutils.TypeExtractor.getDeclaredField( + | org.apache.flink.table.functions.aggfunctions.DistinctAccumulator.class, + | "distinctValueMap"); + | + | + | private final org.apache.flink.table.runtime.aggregate.SingleElementIterable accIt0 = + | new org.apache.flink.table.runtime.aggregate.SingleElementIterable(); + | + | public $distinctCountFuncName() throws Exception { + | + | count = (org.apache.flink.table.functions.aggfunctions.CountAggFunction) + | ${classOf[EncodingUtils].getCanonicalName}.decodeStringToObject( + | "$distinctCountAggFunction", + | ${classOf[UserDefinedFunction].getCanonicalName}.class); + | + | distinctValueMap.setAccessible(true); + | } + | + | public void open(org.apache.flink.api.common.functions.RuntimeContext ctx) { + | org.apache.flink.api.common.state.StateDescriptor acc0_distinctValueMap_dataview_desc = + | (org.apache.flink.api.common.state.StateDescriptor) + | ${classOf[EncodingUtils].getCanonicalName}.decodeStringToObject( + | "$distinctCountDescriptor", + | ${classOf[StateDescriptor[_, _]].getCanonicalName}.class, + | ctx.getUserCodeClassLoader()); + | acc0_distinctValueMap_dataview = new org.apache.flink.table.dataview.StateMapView( + | ctx.getMapState((org.apache.flink.api.common.state.MapStateDescriptor) + | acc0_distinctValueMap_dataview_desc)); + | } + | + | public final void setAggregationResults( + | org.apache.flink.types.Row accs, + | org.apache.flink.types.Row output) { + | + | org.apache.flink.table.functions.AggregateFunction baseClass0 = + | (org.apache.flink.table.functions.AggregateFunction) + | count; + | + | org.apache.flink.table.functions.aggfunctions.DistinctAccumulator distinctAcc0 = + | (org.apache.flink.table.functions.aggfunctions.DistinctAccumulator) accs.getField(0); + | org.apache.flink.table.functions.aggfunctions.CountAccumulator acc0 = + | (org.apache.flink.table.functions.aggfunctions.CountAccumulator) + | distinctAcc0.getRealAcc(); + | + | output.setField(1, baseClass0.getValue(acc0)); + | } + | + | public final void accumulate( + | org.apache.flink.types.Row accs, + | org.apache.flink.types.Row input) throws Exception { + | + | org.apache.flink.table.functions.aggfunctions.DistinctAccumulator distinctAcc0 = + | (org.apache.flink.table.functions.aggfunctions.DistinctAccumulator) accs.getField(0); + | + | distinctValueMap.set(distinctAcc0, acc0_distinctValueMap_dataview); + | + | if (distinctAcc0.add( + | org.apache.flink.types.Row.of((java.lang.Integer) input.getField(1)))) { + | org.apache.flink.table.functions.aggfunctions.CountAccumulator acc0 = + | (org.apache.flink.table.functions.aggfunctions.CountAccumulator) + | distinctAcc0.getRealAcc(); + | + | + | count.accumulate(acc0, (java.lang.Integer) input.getField(1)); + | } + | } + | + | public final void retract( + | org.apache.flink.types.Row accs, + | org.apache.flink.types.Row input) throws Exception { + | + | org.apache.flink.table.functions.aggfunctions.DistinctAccumulator distinctAcc0 = + | (org.apache.flink.table.functions.aggfunctions.DistinctAccumulator) accs.getField(0); + | + | distinctValueMap.set(distinctAcc0, acc0_distinctValueMap_dataview); + | + | if (distinctAcc0.remove( + | org.apache.flink.types.Row.of((java.lang.Integer) input.getField(1)))) { + | org.apache.flink.table.functions.aggfunctions.CountAccumulator acc0 = + | (org.apache.flink.table.functions.aggfunctions.CountAccumulator) + | distinctAcc0.getRealAcc(); + | + | count.retract(acc0 , (java.lang.Integer) input.getField(1)); + | } + | } + | + | public final org.apache.flink.types.Row createAccumulators() + | { + | + | org.apache.flink.types.Row accs = new org.apache.flink.types.Row(1); + | + | org.apache.flink.table.functions.aggfunctions.CountAccumulator acc0 = + | (org.apache.flink.table.functions.aggfunctions.CountAccumulator) + | count.createAccumulator(); + | org.apache.flink.table.functions.aggfunctions.DistinctAccumulator distinctAcc0 = + | (org.apache.flink.table.functions.aggfunctions.DistinctAccumulator) + | new org.apache.flink.table.functions.aggfunctions.DistinctAccumulator (acc0); + | accs.setField( + | 0, + | distinctAcc0); + | + | return accs; + | } + | + | public final void setForwardedFields( + | org.apache.flink.types.Row input, + | org.apache.flink.types.Row output) + | { + | + | output.setField( + | 0, + | input.getField(0)); + | } + | | public final void setConstantFlags(org.apache.flink.types.Row output) | { | @@ -304,10 +476,8 @@ class HarnessTestBase { | org.apache.flink.types.Row accs) { | } | - | public void open(org.apache.flink.api.common.functions.RuntimeContext ctx) { - | } - | | public void cleanup() { + | acc0_distinctValueMap_dataview.clear(); | } | | public void close() { @@ -315,12 +485,11 @@ class HarnessTestBase { |} |""".stripMargin - - protected val minMaxFuncName = "MinMaxAggregateHelper" - protected val sumFuncName = "SumAggregationHelper" - protected val genMinMaxAggFunction = GeneratedAggregationsFunction(minMaxFuncName, minMaxCode) protected val genSumAggFunction = GeneratedAggregationsFunction(sumFuncName, sumAggCode) + protected val genDistinctCountAggFunction = GeneratedAggregationsFunction( + distinctCountFuncName, + distinctCountAggCode) def createHarnessTester[IN, OUT, KEY]( operator: OneInputStreamOperator[IN, OUT], diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala index 304dbb317f358..1706fc8a112e7 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala @@ -20,6 +20,7 @@ package org.apache.flink.table.runtime.stream import java.lang.{Integer => JInt, Long => JLong} import java.math.BigDecimal +import java.sql.Timestamp import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo @@ -661,6 +662,51 @@ class TimeAttributesITCase extends AbstractTestBase { ) assertEquals(expected.sorted, StreamITCase.testResults.sorted) } + + @Test + def testMaterializedRowtimeFilter(): Unit = { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setParallelism(1) + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) + val tEnv = TableEnvironment.getTableEnvironment(env) + StreamITCase.clear + + val data = new mutable.MutableList[(String, Timestamp, Int)] + data.+=(("ACME", new Timestamp(1000L), 12)) + data.+=(("ACME", new Timestamp(2000L), 17)) + data.+=(("ACME", new Timestamp(3000L), 13)) + data.+=(("ACME", new Timestamp(4000L), 11)) + + val t = env.fromCollection(data) + .assignAscendingTimestamps(e => e._2.toInstant.toEpochMilli) + .toTable(tEnv, 'symbol, 'tstamp.rowtime, 'price) + tEnv.registerTable("Ticker", t) + + val sqlQuery = + s""" + |SELECT * + |FROM ( + | SELECT symbol, SUM(price) as price, + | TUMBLE_ROWTIME(tstamp, interval '1' second) as rowTime, + | TUMBLE_START(tstamp, interval '1' second) as startTime, + | TUMBLE_END(tstamp, interval '1' second) as endTime + | FROM Ticker + | GROUP BY symbol, TUMBLE(tstamp, interval '1' second) + |) + |WHERE startTime < endTime + |""".stripMargin + + val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row] + result.addSink(new StreamITCase.StringSink[Row]) + env.execute() + + val expected = List( + "ACME,12,1970-01-01 00:00:01.999,1970-01-01 00:00:01.0,1970-01-01 00:00:02.0", + "ACME,17,1970-01-01 00:00:02.999,1970-01-01 00:00:02.0,1970-01-01 00:00:03.0", + "ACME,13,1970-01-01 00:00:03.999,1970-01-01 00:00:03.0,1970-01-01 00:00:04.0", + "ACME,11,1970-01-01 00:00:04.999,1970-01-01 00:00:04.0,1970-01-01 00:00:05.0") + assertEquals(expected.sorted, StreamITCase.testResults.sorted) + } } object TimeAttributesITCase { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala index c3da65f887a94..ddc2a68754182 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala @@ -78,6 +78,7 @@ class SqlITCase extends StreamingWithStateTestBase { val sqlQuery = "SELECT c, " + " COUNT(DISTINCT b)," + + " SUM(DISTINCT b)," + " SESSION_END(rowtime, INTERVAL '0.005' SECOND) " + "FROM MyTable " + "GROUP BY SESSION(rowtime, INTERVAL '0.005' SECOND), c " @@ -87,9 +88,9 @@ class SqlITCase extends StreamingWithStateTestBase { env.execute() val expected = Seq( - "Hello World,1,1970-01-01 00:00:00.014", // window starts at [9L] till {14L} - "Hello,1,1970-01-01 00:00:00.021", // window starts at [16L] till {21L}, not merged - "Hello,3,1970-01-01 00:00:00.015" // window starts at [1L,2L], + "Hello World,1,9,1970-01-01 00:00:00.014", // window starts at [9L] till {14L} + "Hello,1,16,1970-01-01 00:00:00.021", // window starts at [16L] till {21L}, not merged + "Hello,3,6,1970-01-01 00:00:00.015" // window starts at [1L,2L], // merged with [8L,10L], by [4L], till {15L} ) assertEquals(expected.sorted, StreamITCase.testResults.sorted) @@ -263,6 +264,43 @@ class SqlITCase extends StreamingWithStateTestBase { assertEquals(expected.sorted, StreamITCase.retractedResults.sorted) } + @Test + def testDistinctWithRetraction(): Unit = { + + val env = StreamExecutionEnvironment.getExecutionEnvironment + val tEnv = TableEnvironment.getTableEnvironment(env) + StreamITCase.clear + + val data = new mutable.MutableList[(Int, Long, String)] + data.+=((1, 1L, "Hi")) + data.+=((1, 1L, "Hi World")) + data.+=((1, 1L, "Test")) + data.+=((2, 1L, "Hi World")) + data.+=((2, 1L, "Test")) + data.+=((3, 1L, "Hi World")) + data.+=((3, 1L, "Hi World")) + data.+=((3, 1L, "Hi World")) + data.+=((4, 1L, "Hi World")) + data.+=((4, 1L, "Test")) + + val t = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c) + tEnv.registerTable("MyTable", t) + + // "1,1,3", "2,1,2", "3,1,1", "4,1,2" + val distinct = "SELECT a, COUNT(DISTINCT b) AS distinct_b, COUNT(DISTINCT c) AS distinct_c " + + "FROM MyTable GROUP BY a" + val nestedDistinct = s"SELECT distinct_b, COUNT(DISTINCT distinct_c) " + + s"FROM ($distinct) GROUP BY distinct_b" + + val result = tEnv.sqlQuery(nestedDistinct).toRetractStream[Row] + result.addSink(new StreamITCase.RetractingSink).setParallelism(1) + + env.execute() + + val expected = List("1,3") + assertEquals(expected.sorted, StreamITCase.retractedResults.sorted) + } + @Test def testUnboundedGroupByCollect(): Unit = { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala index df0f01be0d58a..0fb175370fbfe 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala @@ -127,8 +127,6 @@ class TemporalJoinITCase extends StreamingWithStateTestBase { var expectedOutput = new mutable.HashSet[String]() expectedOutput += (2 * 114).toString - expectedOutput += (1 * 102).toString - expectedOutput += (50 * 1).toString expectedOutput += (3 * 116).toString val orders = env @@ -142,11 +140,15 @@ class TemporalJoinITCase extends StreamingWithStateTestBase { tEnv.registerTable("Orders", orders) tEnv.registerTable("RatesHistory", ratesHistory) + tEnv.registerTable("FilteredRatesHistory", tEnv.scan("RatesHistory").filter('rate > 110L)) tEnv.registerFunction( "Rates", - ratesHistory.createTemporalTableFunction('rowtime, 'currency)) + tEnv.scan("FilteredRatesHistory").createTemporalTableFunction('rowtime, 'currency)) + tEnv.registerTable("TemporalJoinResult", tEnv.sqlQuery(sqlQuery)) - val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row] + // Scan from registered table to test for interplay between + // LogicalCorrelateToTemporalTableJoinRule and TableScanRule + val result = tEnv.scan("TemporalJoinResult").toAppendStream[Row] result.addSink(new StreamITCase.StringSink[Row]) env.execute() diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala index f4b1f8059741d..60e03d4cd2460 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala @@ -284,6 +284,27 @@ class CorrelateITCase extends AbstractTestBase { ) } + @Test + def testTableFunctionCollectorInit(): Unit = { + val t = testData(env).toTable(tEnv).as('a, 'b, 'c) + val func0 = new TableFunc0 + + // this case will generate 'timestamp' member field and 'DateFormatter' + val result = t + .join(func0('c) as('d, 'e)) + .where(dateFormat(currentTimestamp(), "yyyyMMdd") === 'd) + .select('c, 'd, 'e) + .toAppendStream[Row] + + result.addSink(new StreamITCase.StringSink[Row]) + env.execute() + + assertEquals( + Seq(), + StreamITCase.testResults.sorted + ) + } + private def testData( env: StreamExecutionEnvironment) : DataStream[(Int, Long, String)] = { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala index 197a039128818..7060d4db2a7c0 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala @@ -27,6 +27,7 @@ import org.junit.Assert._ import org.junit.Test import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.table.expressions.utils.Func20 import org.apache.flink.table.expressions.{Literal, Null} import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, WeightedAvg} @@ -225,7 +226,12 @@ class JoinITCase extends StreamingWithStateTestBase { val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c) val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h) - val joinT = ds1.join(ds2).where('b === 'e).select('c, 'g) + val testOpenCall = new Func20 + + val joinT = ds1.join(ds2) + .where('b === 'e) + .where(testOpenCall('a + 'd)) + .select('c, 'g) val expected = Seq("Hi,Hallo", "Hello,Hallo Welt", "Hello world,Hallo Welt") val results = joinT.toRetractStream[Row] diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala index 7483b04d9cac1..055501a6a0160 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala @@ -18,8 +18,20 @@ package org.apache.flink.table.runtime.types -import org.apache.flink.util.TestLogger -import org.junit.Test +import org.apache.flink.api.common.state.{ListState, ListStateDescriptor} +import org.apache.flink.api.common.typeinfo.Types +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.configuration.Configuration +import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend +import org.apache.flink.streaming.api.functions.KeyedProcessFunction +import org.apache.flink.streaming.api.operators.{AbstractStreamOperator, KeyedProcessOperator} +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord +import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness +import org.apache.flink.types.Row +import org.apache.flink.util.{Collector, InstantiationUtil, TestLogger} + +import org.junit.{Assert, Test} class CRowSerializerTest extends TestLogger { @@ -29,6 +41,70 @@ class CRowSerializerTest extends TestLogger { @Test def testDefaultConstructor(): Unit = { new CRowSerializer.CRowSerializerConfigSnapshot() + + InstantiationUtil.instantiate(classOf[CRowSerializer.CRowSerializerConfigSnapshot]) + } + + @Test + def testStateRestore(): Unit = { + + class IKeyedProcessFunction extends KeyedProcessFunction[Integer, Integer, Integer] { + var state: ListState[CRow] = _ + override def open(parameters: Configuration): Unit = { + val stateDesc = new ListStateDescriptor[CRow]("CRow", + new CRowTypeInfo(new RowTypeInfo(Types.INT))) + state = getRuntimeContext.getListState(stateDesc) + } + override def processElement(value: Integer, + ctx: KeyedProcessFunction[Integer, Integer, Integer]#Context, + out: Collector[Integer]): Unit = { + state.add(new CRow(Row.of(value), true)) + } + } + + val operator = new KeyedProcessOperator[Integer, Integer, Integer](new IKeyedProcessFunction) + + var testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, Integer, Integer]( + operator, + new KeySelector[Integer, Integer] { + override def getKey(value: Integer): Integer= -1 + }, + Types.INT, 1, 1, 0) + testHarness.setup() + testHarness.open() + + testHarness.processElement(new StreamRecord[Integer](1, 1L)) + testHarness.processElement(new StreamRecord[Integer](2, 1L)) + testHarness.processElement(new StreamRecord[Integer](3, 1L)) + + Assert.assertEquals(1, numKeyedStateEntries(operator)) + + val snapshot = testHarness.snapshot(0L, 0L) + testHarness.close() + + testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, Integer, Integer]( + operator, + new KeySelector[Integer, Integer] { + override def getKey(value: Integer): Integer= -1 + }, + Types.INT, 1, 1, 0) + testHarness.setup() + + testHarness.initializeState(snapshot) + + testHarness.open() + + Assert.assertEquals(1, numKeyedStateEntries(operator)) + + testHarness.close() + } + + def numKeyedStateEntries(operator: AbstractStreamOperator[_]): Int = { + val keyedStateBackend = operator.getKeyedStateBackend + keyedStateBackend match { + case hksb: HeapKeyedStateBackend[_] => hksb.numKeyValueStateEntries + case _ => throw new UnsupportedOperationException + } } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala index 64fcc8ac7c480..1209595837bf3 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala @@ -85,7 +85,9 @@ object CommonTestData { .withSchema(schemaDesc1) if (isStreaming) { - externalTableBuilder1.inAppendMode() + externalTableBuilder1.supportsStreaming().inAppendMode() + } else { + externalTableBuilder1.supportsBatch() } val csvRecord2 = Seq( @@ -126,7 +128,9 @@ object CommonTestData { .withSchema(schemaDesc2) if (isStreaming) { - externalTableBuilder2.inAppendMode() + externalTableBuilder2.supportsStreaming().inAppendMode() + } else { + externalTableBuilder2.supportsBatch() } val tempFilePath3 = writeToTempFile("", "csv-test3", "tmp") @@ -145,7 +149,9 @@ object CommonTestData { .withSchema(schemaDesc3) if (isStreaming) { - externalTableBuilder3.inAppendMode() + externalTableBuilder3.supportsStreaming().inAppendMode() + } else { + externalTableBuilder3.supportsBatch() } val catalog = new InMemoryExternalCatalog("test") diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java index 426a891e81423..93f362573cc88 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java @@ -33,6 +33,7 @@ public class MesosOptions { /** * The initial number of Mesos tasks to allocate. */ + @Deprecated public static final ConfigOption INITIAL_TASKS = key("mesos.initial-tasks") .defaultValue(0) @@ -47,6 +48,7 @@ public class MesosOptions { * *

    By default, we take the number of initially requested tasks. */ + @Deprecated public static final ConfigOption MAX_FAILED_TASKS = key("mesos.maximum-failed-tasks") .defaultValue(-1) @@ -136,8 +138,9 @@ public class MesosOptions { /** * Config parameter to configure which configuration keys will dynamically get a port assigned through Mesos. */ - public static final ConfigOption PORT_ASSIGNMENTS = key("mesos.resourcemanager.tasks.port-assignments") - .defaultValue("") + public static final ConfigOption PORT_ASSIGNMENTS = + key("mesos.resourcemanager.tasks.port-assignments") + .noDefaultValue() .withDescription(Description.builder() .text("Comma-separated list of configuration keys which represent a configurable port. " + "All port keys will dynamically get a port assigned through Mesos.") diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java index 84ec2229a2ad4..637442c899d31 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; @@ -64,12 +65,13 @@ public class LaunchableMesosWorker implements LaunchableTask { protected static final Logger LOG = LoggerFactory.getLogger(LaunchableMesosWorker.class); + /** * The set of configuration keys to be dynamically configured with a port allocated from Mesos. */ - static final String[] TM_PORT_KEYS = { + static final Set TM_PORT_KEYS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( "taskmanager.rpc.port", - "taskmanager.data.port"}; + "taskmanager.data.port"))); private final MesosArtifactResolver resolver; private final ContainerSpecification containerSpec; @@ -342,16 +344,18 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, MesosResourceAllocation al * @return A deterministically ordered Set of port keys to expose from the TM container */ static Set extractPortKeys(Configuration config) { - final LinkedHashSet tmPortKeys = new LinkedHashSet<>(Arrays.asList(TM_PORT_KEYS)); + final LinkedHashSet tmPortKeys = new LinkedHashSet<>(TM_PORT_KEYS); final String portKeys = config.getString(PORT_ASSIGNMENTS); - Arrays.stream(portKeys.split(",")) - .map(String::trim) - .peek(key -> LOG.debug("Adding port key " + key + " to mesos request")) - .forEach(tmPortKeys::add); + if (portKeys != null) { + Arrays.stream(portKeys.split(",")) + .map(String::trim) + .peek(key -> LOG.debug("Adding port key {} to mesos request")) + .forEach(tmPortKeys::add); + } - return tmPortKeys; + return Collections.unmodifiableSet(tmPortKeys); } @Override diff --git a/flink-mesos/src/main/resources/META-INF/NOTICE b/flink-mesos/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..145489a7a99d5 --- /dev/null +++ b/flink-mesos/src/main/resources/META-INF/NOTICE @@ -0,0 +1,18 @@ +flink-mesos +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.netflix.fenzo:fenzo-core:0.10.1 +- org.apache.mesos:mesos:1.0.1 +- com.fasterxml.jackson.core:jackson-annotations:2.4.0 +- com.fasterxml.jackson.core:jackson-core:2.4.5 +- com.fasterxml.jackson.core:jackson-databind:2.4.5 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.google.protobuf:protobuf-java:2.6.2 diff --git a/flink-mesos/src/main/resources/META-INF/licenses/LICENSE.protobuf b/flink-mesos/src/main/resources/META-INF/licenses/LICENSE.protobuf new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/flink-mesos/src/main/resources/META-INF/licenses/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorkerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorkerTest.java index 6784e427c1f59..48a436cb9954f 100644 --- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorkerTest.java +++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorkerTest.java @@ -23,11 +23,14 @@ import org.junit.Test; -import java.util.Iterator; +import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import static org.apache.flink.mesos.configuration.MesosOptions.PORT_ASSIGNMENTS; -import static org.junit.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; /** * Test that mesos config are extracted correctly from the configuration. @@ -36,20 +39,31 @@ public class LaunchableMesosWorkerTest extends TestLogger { @Test public void canGetPortKeys() { + // Setup + Set additionalPorts = new HashSet<>(Arrays.asList("someport.here", "anotherport")); + + Configuration config = new Configuration(); + config.setString(PORT_ASSIGNMENTS, String.join(",", additionalPorts)); + + // Act + Set portKeys = LaunchableMesosWorker.extractPortKeys(config); + + // Assert + Set expectedPorts = new HashSet<>(LaunchableMesosWorker.TM_PORT_KEYS); + expectedPorts.addAll(additionalPorts); + assertThat(portKeys, is(equalTo(expectedPorts))); + } + + @Test + public void canGetNoPortKeys() { // Setup Configuration config = new Configuration(); - config.setString(PORT_ASSIGNMENTS, "someport.here,anotherport"); // Act Set portKeys = LaunchableMesosWorker.extractPortKeys(config); // Assert - assertEquals("Must get right number of port keys", 4, portKeys.size()); - Iterator iterator = portKeys.iterator(); - assertEquals("port key must be correct", LaunchableMesosWorker.TM_PORT_KEYS[0], iterator.next()); - assertEquals("port key must be correct", LaunchableMesosWorker.TM_PORT_KEYS[1], iterator.next()); - assertEquals("port key must be correct", "someport.here", iterator.next()); - assertEquals("port key must be correct", "anotherport", iterator.next()); + assertThat(portKeys, is(equalTo(LaunchableMesosWorker.TM_PORT_KEYS))); } } diff --git a/flink-metrics/flink-metrics-core/src/main/java/org/apache/flink/metrics/MeterView.java b/flink-metrics/flink-metrics-core/src/main/java/org/apache/flink/metrics/MeterView.java index 8df0e868f8f8e..6d53325edafe3 100644 --- a/flink-metrics/flink-metrics-core/src/main/java/org/apache/flink/metrics/MeterView.java +++ b/flink-metrics/flink-metrics-core/src/main/java/org/apache/flink/metrics/MeterView.java @@ -50,7 +50,11 @@ public MeterView(int timeSpanInSeconds) { public MeterView(Counter counter, int timeSpanInSeconds) { this.counter = counter; - this.timeSpanInSeconds = timeSpanInSeconds - (timeSpanInSeconds % UPDATE_INTERVAL_SECONDS); + // the time-span must be larger than the update-interval as otherwise the array has a size of 1, + // for which no rate can be computed as no distinct before/after measurement exists. + this.timeSpanInSeconds = Math.max( + timeSpanInSeconds - (timeSpanInSeconds % UPDATE_INTERVAL_SECONDS), + UPDATE_INTERVAL_SECONDS); this.values = new long[this.timeSpanInSeconds / UPDATE_INTERVAL_SECONDS + 1]; } diff --git a/flink-metrics/flink-metrics-core/src/test/java/org/apache/flink/metrics/MeterViewTest.java b/flink-metrics/flink-metrics-core/src/test/java/org/apache/flink/metrics/MeterViewTest.java index a7a63b099493a..7752a6e80bb32 100644 --- a/flink-metrics/flink-metrics-core/src/test/java/org/apache/flink/metrics/MeterViewTest.java +++ b/flink-metrics/flink-metrics-core/src/test/java/org/apache/flink/metrics/MeterViewTest.java @@ -20,6 +20,7 @@ import org.junit.Test; +import static org.apache.flink.metrics.View.UPDATE_INTERVAL_SECONDS; import static org.junit.Assert.assertEquals; /** @@ -94,4 +95,14 @@ public void testGetRate() { assertEquals(0.0, m.getRate(), 0.1); // 480 - 480 / 60 } + + @Test + public void testTimeSpanBelowUpdateRate() { + int timeSpanInSeconds = 1; + MeterView m = new MeterView(timeSpanInSeconds); + assert timeSpanInSeconds < UPDATE_INTERVAL_SECONDS; + m.markEvent(); + m.update(); + assertEquals(0.2, m.getRate(), 0.0); + } } diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java index b7e1c24fa6034..5462b32354900 100644 --- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java +++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java @@ -118,6 +118,7 @@ public void close() { public void report() { DatadogHttpRequest request = new DatadogHttpRequest(); + List gaugesToRemove = new ArrayList<>(); for (Map.Entry entry : gauges.entrySet()) { DGauge g = entry.getValue(); try { @@ -125,11 +126,19 @@ public void report() { // Flink uses Gauge to store many types other than Number g.getMetricValue(); request.addGauge(g); + } catch (ClassCastException e) { + LOGGER.info("The metric {} will not be reported because only number types are supported by this reporter.", g.getMetric()); + gaugesToRemove.add(entry.getKey()); } catch (Exception e) { - // Remove that Gauge if it's not of Number type - gauges.remove(entry.getKey()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("The metric {} will not be reported because it threw an exception.", g.getMetric(), e); + } else { + LOGGER.info("The metric {} will not be reported because it threw an exception.", g.getMetric()); + } + gaugesToRemove.add(entry.getKey()); } } + gaugesToRemove.forEach(gauges::remove); for (DCounter c : counters.values()) { request.addCounter(c); diff --git a/flink-metrics/flink-metrics-datadog/src/main/resources/META-INF/NOTICE b/flink-metrics/flink-metrics-datadog/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..e577eda4c2294 --- /dev/null +++ b/flink-metrics/flink-metrics-datadog/src/main/resources/META-INF/NOTICE @@ -0,0 +1,10 @@ +flink-metrics-datadog +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.squareup.okhttp3:okhttp:3.7.0 +- com.squareup.okio:okio:1.12.0 diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml index 4ef94264346c4..b84aeed9e8b9c 100644 --- a/flink-metrics/flink-metrics-dropwizard/pom.xml +++ b/flink-metrics/flink-metrics-dropwizard/pom.xml @@ -84,27 +84,4 @@ under the License. - - - - - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - diff --git a/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/ganglia/GangliaReporter.java b/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/ganglia/GangliaReporter.java deleted file mode 100644 index 87199011fc563..0000000000000 --- a/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/ganglia/GangliaReporter.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.metrics.ganglia; - -import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.dropwizard.ScheduledDropwizardReporter; -import org.apache.flink.metrics.MetricConfig; - -import com.codahale.metrics.ScheduledReporter; -import info.ganglia.gmetric4j.gmetric.GMetric; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -/** - * This class acts as a factory for the {@link com.codahale.metrics.ganglia.GangliaReporter} and allows using it as a - * Flink reporter. - */ -@PublicEvolving -public class GangliaReporter extends ScheduledDropwizardReporter { - - public static final String ARG_DMAX = "dmax"; - public static final String ARG_TMAX = "tmax"; - public static final String ARG_TTL = "ttl"; - public static final String ARG_MODE_ADDRESSING = "addressingMode"; - - @Override - public ScheduledReporter getReporter(MetricConfig config) { - - try { - String host = config.getString(ARG_HOST, null); - int port = config.getInteger(ARG_PORT, -1); - if (host == null || host.length() == 0 || port < 1) { - throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port); - } - String addressingMode = config.getString(ARG_MODE_ADDRESSING, "MULTICAST"); - int ttl = config.getInteger(ARG_TTL, 1); - GMetric gMetric = new GMetric(host, port, GMetric.UDPAddressingMode.valueOf(addressingMode), ttl); - - String prefix = config.getString(ARG_PREFIX, null); - String conversionRate = config.getString(ARG_CONVERSION_RATE, null); - String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null); - int dMax = config.getInteger(ARG_DMAX, 0); - int tMax = config.getInteger(ARG_TMAX, 60); - - com.codahale.metrics.ganglia.GangliaReporter.Builder builder = - com.codahale.metrics.ganglia.GangliaReporter.forRegistry(registry); - - if (prefix != null) { - builder.prefixedWith(prefix); - } - if (conversionRate != null) { - builder.convertRatesTo(TimeUnit.valueOf(conversionRate)); - } - if (conversionDuration != null) { - builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration)); - } - builder.withDMax(dMax); - builder.withTMax(tMax); - - log.info("Configured GangliaReporter with {host:{}, port:{}, dmax:{}, tmax:{}, ttl:{}, addressingMode:{}}", - host, port, dMax, tMax, ttl, addressingMode); - return builder.build(gMetric); - } catch (IOException e) { - throw new RuntimeException("Error while instantiating GangliaReporter.", e); - } - } -} diff --git a/flink-metrics/flink-metrics-graphite/pom.xml b/flink-metrics/flink-metrics-graphite/pom.xml index eeb7f8b08446c..37cee5e1ddc55 100644 --- a/flink-metrics/flink-metrics-graphite/pom.xml +++ b/flink-metrics/flink-metrics-graphite/pom.xml @@ -69,20 +69,24 @@ under the License. - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - + org.apache.maven.plugins + maven-shade-plugin - make-assembly + shade-flink package - single + shade + + + + org.apache.flink:flink-metrics-dropwizard + io.dropwizard.metrics:metrics-core + io.dropwizard.metrics:metrics-graphite + + + diff --git a/flink-metrics/flink-metrics-graphite/src/main/resources/META-INF/NOTICE b/flink-metrics/flink-metrics-graphite/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..ef94ebed7582e --- /dev/null +++ b/flink-metrics/flink-metrics-graphite/src/main/resources/META-INF/NOTICE @@ -0,0 +1,10 @@ +flink-metrics-graphite +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.dropwizard.metrics:metrics-core:3.1.5 +- io.dropwizard.metrics:metrics-graphite:3.1.5 diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml index 750ab620a2f6b..ccf90ef07375a 100644 --- a/flink-metrics/flink-metrics-jmx/pom.xml +++ b/flink-metrics/flink-metrics-jmx/pom.xml @@ -29,7 +29,7 @@ under the License. .. - flink-metrics-jmx + flink-metrics-jmx_${scala.binary.version} flink-metrics-jmx diff --git a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java index 915c02b644d8f..036796c582d45 100644 --- a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java +++ b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java @@ -107,7 +107,7 @@ public void testJobManagerJMXMetricAccess() throws Exception { client.setDetached(true); client.submitJob(jobGraph, JMXJobManagerMetricTest.class.getClassLoader()); - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> client.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, diff --git a/flink-metrics/flink-metrics-prometheus/pom.xml b/flink-metrics/flink-metrics-prometheus/pom.xml index 145113d9999eb..430753e8d1e5e 100644 --- a/flink-metrics/flink-metrics-prometheus/pom.xml +++ b/flink-metrics/flink-metrics-prometheus/pom.xml @@ -29,7 +29,7 @@ under the License. .. - flink-metrics-prometheus + flink-metrics-prometheus_${scala.binary.version} flink-metrics-prometheus diff --git a/flink-metrics/flink-metrics-prometheus/src/main/resources/META-INF/NOTICE b/flink-metrics/flink-metrics-prometheus/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..dedb042af855e --- /dev/null +++ b/flink-metrics/flink-metrics-prometheus/src/main/resources/META-INF/NOTICE @@ -0,0 +1,12 @@ +flink-metrics-prometheus +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.prometheus:simpleclient:0.3.0 +- io.prometheus:simpleclient_common:0.3.0 +- io.prometheus:simpleclient_httpserver:0.3.0 +- io.prometheus:simpleclient_pushgateway:0.3.0 diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml index f4031b1288c3f..403f5162ebe3b 100644 --- a/flink-metrics/pom.xml +++ b/flink-metrics/pom.xml @@ -36,7 +36,6 @@ under the License. flink-metrics-core flink-metrics-dropwizard - flink-metrics-ganglia flink-metrics-graphite flink-metrics-jmx flink-metrics-prometheus diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index e99a28b36a212..eb6b3ef8c50e8 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -836,9 +836,11 @@ public void processElement(Tuple2 value, Context ctx, Collector value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get(key); - assertEquals("Key mismatch", key, value.f0.intValue()); - if (expected == value.f1) { + Tuple2 value = + future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get(key); + + if (value != null && value.f0 != null && expected == value.f1) { + assertEquals("Key mismatch", key, value.f0.intValue()); success = true; } else { // Retry @@ -1271,7 +1273,7 @@ public void close() throws Exception { // Free cluster resources clusterClient.cancel(jobId); // cancel() is non-blocking so do this to make sure the job finished - CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + CompletableFuture jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobId), Time.milliseconds(50), deadline, diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java index 9116304ebf5a5..36637c0d34200 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java @@ -44,6 +44,7 @@ public class NonHAQueryableStateFsBackendITCase extends AbstractQueryableStateTe // we always use all TaskManagers so that the JM oracle is always properly re-registered private static final int NUM_TMS = 2; private static final int NUM_SLOTS_PER_TM = 2; + private static final int NUM_PORT_COUNT = 100; private static final int QS_PROXY_PORT_RANGE_START = 9084; private static final int QS_SERVER_PORT_RANGE_START = 9089; @@ -86,10 +87,10 @@ private static Configuration getConfig() { config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1); config.setString( QueryableStateOptions.PROXY_PORT_RANGE, - QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS)); + QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_PORT_COUNT)); config.setString( QueryableStateOptions.SERVER_PORT_RANGE, - QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS)); + QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_PORT_COUNT)); config.setBoolean(WebOptions.SUBMIT_ENABLE, false); return config; } diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml index f4c4d0a19655a..2f381649c87e9 100644 --- a/flink-runtime-web/pom.xml +++ b/flink-runtime-web/pom.xml @@ -128,6 +128,9 @@ under the License. web/** + + src/main/resources + diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java index 679dfe988ef5b..46baa2696505d 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java @@ -34,8 +34,6 @@ import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.FlinkException; -import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus; - import javax.annotation.Nonnull; import java.net.InetSocketAddress; @@ -108,13 +106,7 @@ protected CompletableFuture handleRequest( }); return jobSubmissionFuture - .thenCombine(jarUploadFuture, (ack, jobGraph) -> new JarRunResponseBody(jobGraph.getJobID())) - .exceptionally(throwable -> { - throw new CompletionException(new RestHandlerException( - throwable.getMessage(), - HttpResponseStatus.INTERNAL_SERVER_ERROR, - throwable)); - }); + .thenCombine(jarUploadFuture, (ack, jobGraph) -> new JarRunResponseBody(jobGraph.getJobID())); } private SavepointRestoreSettings getSavepointRestoreSettings( diff --git a/flink-runtime-web/src/main/resources/META-INF/NOTICE b/flink-runtime-web/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..32506e6c888b4 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/NOTICE @@ -0,0 +1,40 @@ +flink-runtime-web +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- nvd3#1.8.4 + +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) +See bundled license files for details. + +- angular:1.4.8 +- angular-drag-and-drop-list:1.4.0 +- angular-moment:0.10.3 +- angular-ui-router:0.2.15 +- bootstrap:3.3.6 +- dagre:0.7.5 +- dagre-d3:0.4.17 +- ev-emitter:1.1.1 +- font-awesome:4.5.0 (CSS) +- graphlib:1.0.7 +- imagesloaded:4.1.4 +- jquery:2.2.0 +- lodash:3.10.1 +- moment:2.10.6 +- moment-duration-format:1.3.0 +- qtip2:2.2.1 +- Split.js:1.0.6 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- d3:3.5.12 + +This project bundles the following dependencies under SIL OFL 1.1 license. (https://opensource.org/licenses/OFL-1.1) +See bundled license files for details. + +- font-awesome:4.5.0 (Font) diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular new file mode 100644 index 0000000000000..91f0644936811 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2010-2018 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-drag-and-drop-list b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-drag-and-drop-list new file mode 100644 index 0000000000000..1787fc65befe1 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-drag-and-drop-list @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Marcel Juenemann +Copyright (c) 2014-2016 Google Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-moment b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-moment new file mode 100644 index 0000000000000..fe9db79b9113a --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-moment @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Uri Shaked and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-ui-router b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-ui-router new file mode 100644 index 0000000000000..6413b092d70f7 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.angular-ui-router @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.bootstrap b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.bootstrap new file mode 100644 index 0000000000000..6ca0ceecc5d79 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.bootstrap @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2011-2018 Twitter, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.d3 b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.d3 new file mode 100644 index 0000000000000..1d9d875edb469 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.d3 @@ -0,0 +1,27 @@ +Copyright 2010-2017 Mike Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre-d3 b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre-d3 new file mode 100644 index 0000000000000..1d64ed68ce64d --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.dagre-d3 @@ -0,0 +1,19 @@ +Copyright (c) 2013 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.ev-emitter b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.ev-emitter new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.ev-emitter @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.font-awesome b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.font-awesome new file mode 100644 index 0000000000000..b9fb2c6e6ef06 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.font-awesome @@ -0,0 +1,97 @@ +The MIT license + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + SIL OPEN FONT LICENSE + +Version 1.1 - 26 February 2007 +PREAMBLE + +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. +DEFINITIONS + +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting — in part or in whole — any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. +PERMISSION & CONDITIONS + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. +TERMINATION + +This license becomes null and void if any of the above conditions are +not met. +DISCLAIMER + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.graphlib b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.graphlib new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.graphlib @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.imagesloaded b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.imagesloaded new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.imagesloaded @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.jquery b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.jquery new file mode 100644 index 0000000000000..e3dbacb999cef --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.jquery @@ -0,0 +1,20 @@ +Copyright JS Foundation and other contributors, https://js.foundation/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.lodash b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.lodash new file mode 100644 index 0000000000000..e3a30b4eafbab --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.lodash @@ -0,0 +1,10 @@ +The MIT License + +Copyright JS Foundation and other contributors + +Based on Underscore.js, copyright Jeremy Ashkenas, +DocumentCloud and Investigative Reporters & Editors + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/lodash/lodash \ No newline at end of file diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment new file mode 100644 index 0000000000000..8618b7333d6f5 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment @@ -0,0 +1,22 @@ +Copyright (c) JS Foundation and other contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment-duration-format b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment-duration-format new file mode 100644 index 0000000000000..06ec6fbe07368 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.moment-duration-format @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 vin-car + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.qtip2 b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.qtip2 new file mode 100644 index 0000000000000..0cfb790dc91d3 --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.qtip2 @@ -0,0 +1,22 @@ +Copyright (c) 2012 Craig Michael Thompson + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.split b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.split new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/flink-runtime-web/src/main/resources/META-INF/licenses/LICENSE.split @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java index 5fc3ff5c06790..1b236357ed589 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java @@ -39,6 +39,8 @@ import java.nio.file.Path; import java.util.Optional; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -90,6 +92,8 @@ public void testRunJar() throws Exception { if (expected.isPresent()) { // implies the job was actually submitted assertTrue(expected.get().getMessage().contains("ProgramInvocationException")); + // original cause is preserved in stack trace + assertThat(expected.get().getMessage(), containsString("ZipException")); // implies the jar was registered for the job graph (otherwise the jar name would not occur in the exception) // implies the jar was uploaded (otherwise the file would not be found at all) assertTrue(expected.get().getMessage().contains("empty.jar'. zip file is empty")); diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index 3412fd4713d59..8ad6440b51ef2 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -567,6 +567,17 @@ under the License. + + io.netty:netty + + + META-INF/license + + META-INF/NOTICE.txt + + * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java index 57337b6286fb1..3890f79ce7d3f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.checkpoint.decline.CheckpointDeclineException; import org.apache.flink.runtime.checkpoint.hooks.MasterHooks; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.execution.ExecutionState; @@ -1249,11 +1250,14 @@ private void discardCheckpoint(PendingCheckpoint pendingCheckpoint, @Nullable Th final long checkpointId = pendingCheckpoint.getCheckpointId(); - final String reason = (cause != null) ? cause.getMessage() : ""; + LOG.info("Discarding checkpoint {} of job {}.", checkpointId, job, cause); - LOG.info("Discarding checkpoint {} of job {} because: {}", checkpointId, job, reason); + if (cause == null || cause instanceof CheckpointDeclineException) { + pendingCheckpoint.abortDeclined(); + } else { + pendingCheckpoint.abortError(cause); + } - pendingCheckpoint.abortDeclined(); rememberRecentCheckpointId(checkpointId); // we don't have to schedule another "dissolving" checkpoint any more because the diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java index 22244f6cb8d51..b8dc554570617 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.rpc.RpcGateway; public interface CheckpointCoordinatorGateway extends RpcGateway { @@ -31,9 +32,5 @@ void acknowledgeCheckpoint( final CheckpointMetrics checkpointMetrics, final TaskStateSnapshot subtaskState); - void declineCheckpoint( - JobID jobID, - ExecutionAttemptID executionAttemptID, - long checkpointId, - Throwable cause); + void declineCheckpoint(DeclineCheckpoint declineCheckpoint); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java index dad45eb669c0f..9e15aebd048fe 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java @@ -18,6 +18,9 @@ package org.apache.flink.runtime.checkpoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.Serializable; import static org.apache.flink.util.Preconditions.checkArgument; @@ -26,6 +29,7 @@ * Counts of checkpoints. */ public class CheckpointStatsCounts implements Serializable { + private static final Logger LOG = LoggerFactory.getLogger(CheckpointStatsCounts.class); private static final long serialVersionUID = -5229425063269482528L; @@ -147,9 +151,8 @@ void incrementInProgressCheckpoints() { * {@link #incrementInProgressCheckpoints()}. */ void incrementCompletedCheckpoints() { - if (--numInProgressCheckpoints < 0) { - throw new IllegalStateException("Incremented the completed number of checkpoints " + - "without incrementing the in progress checkpoints before."); + if (canDecrementOfInProgressCheckpointsNumber()) { + numInProgressCheckpoints--; } numCompletedCheckpoints++; } @@ -161,9 +164,8 @@ void incrementCompletedCheckpoints() { * {@link #incrementInProgressCheckpoints()}. */ void incrementFailedCheckpoints() { - if (--numInProgressCheckpoints < 0) { - throw new IllegalStateException("Incremented the completed number of checkpoints " + - "without incrementing the in progress checkpoints before."); + if (canDecrementOfInProgressCheckpointsNumber()) { + numInProgressCheckpoints--; } numFailedCheckpoints++; } @@ -181,4 +183,14 @@ CheckpointStatsCounts createSnapshot() { numCompletedCheckpoints, numFailedCheckpoints); } + + private boolean canDecrementOfInProgressCheckpointsNumber() { + boolean decrementLeadsToNegativeNumber = numInProgressCheckpoints - 1 < 0; + if (decrementLeadsToNegativeNumber) { + String errorMessage = "Incremented the completed number of checkpoints " + + "without incrementing the in progress checkpoints before."; + LOG.warn(errorMessage); + } + return !decrementLeadsToNegativeNumber; + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java index 1b51ac4bf8d3a..1bc6b0e4baa9f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java @@ -34,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; @@ -433,25 +434,23 @@ public void abortSubsumed() { } } + public void abortDeclined() { - try { - Exception cause = new Exception("Checkpoint was declined (tasks not ready)"); - onCompletionPromise.completeExceptionally(cause); - reportFailedCheckpoint(cause); - } finally { - dispose(true); - } + abortWithCause(new Exception("Checkpoint was declined (tasks not ready)")); } /** * Aborts the pending checkpoint due to an error. * @param cause The error's exception. */ - public void abortError(Throwable cause) { + public void abortError(@Nonnull Throwable cause) { + abortWithCause(new Exception("Checkpoint failed: " + cause.getMessage(), cause)); + } + + private void abortWithCause(@Nonnull Exception cause) { try { - Exception failure = new Exception("Checkpoint failed: " + cause.getMessage(), cause); - onCompletionPromise.completeExceptionally(failure); - reportFailedCheckpoint(failure); + onCompletionPromise.completeExceptionally(cause); + reportFailedCheckpoint(cause); } finally { dispose(true); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java index b0173886d5730..02fc2013fb049 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java @@ -71,13 +71,12 @@ public StateAssignmentOperation( this.allowNonRestoredState = allowNonRestoredState; } - public boolean assignStates() throws Exception { + public void assignStates() { Map localOperators = new HashMap<>(operatorStates); - Map localTasks = this.tasks; checkStateMappingCompleteness(allowNonRestoredState, operatorStates, tasks); - for (Map.Entry task : localTasks.entrySet()) { + for (Map.Entry task : this.tasks.entrySet()) { final ExecutionJobVertex executionJobVertex = task.getValue(); // find the states of all operators belonging to this task @@ -108,7 +107,6 @@ public boolean assignStates() throws Exception { assignAttemptState(task.getValue(), operatorStates); } - return true; } private void assignAttemptState(ExecutionJobVertex executionJobVertex, List operatorStates) { @@ -254,10 +252,6 @@ public static OperatorSubtaskState operatorSubtaskStateFrom( new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); } - private static boolean isHeadOperator(int opIdx, List operatorIDs) { - return opIdx == operatorIDs.size() - 1; - } - public void checkParallelismPreconditions(List operatorStates, ExecutionJobVertex executionJobVertex) { for (OperatorState operatorState : operatorStates) { checkParallelismPreconditions(operatorState, executionJobVertex); @@ -278,19 +272,16 @@ private void reDistributeKeyedStates( for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) { OperatorState operatorState = oldOperatorStates.get(operatorIndex); int oldParallelism = operatorState.getParallelism(); - for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) { OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); - if (isHeadOperator(operatorIndex, newOperatorIDs)) { - Tuple2, List> subKeyedStates = reAssignSubKeyedStates( - operatorState, - newKeyGroupPartitions, - subTaskIndex, - newParallelism, - oldParallelism); - newManagedKeyedState.put(instanceID, subKeyedStates.f0); - newRawKeyedState.put(instanceID, subKeyedStates.f1); - } + Tuple2, List> subKeyedStates = reAssignSubKeyedStates( + operatorState, + newKeyGroupPartitions, + subTaskIndex, + newParallelism, + oldParallelism); + newManagedKeyedState.put(instanceID, subKeyedStates.f0); + newRawKeyedState.put(instanceID, subKeyedStates.f1); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java index faee588c48d7c..fa8407729d90c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java @@ -18,6 +18,8 @@ package org.apache.flink.runtime.checkpoint.savepoint; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.checkpoint.MasterState; import org.apache.flink.runtime.checkpoint.OperatorState; @@ -67,7 +69,9 @@ * +--------------+---------------------+---------+------+---------------+ * */ -class SavepointV2Serializer implements SavepointSerializer { +@Internal +@VisibleForTesting +public class SavepointV2Serializer implements SavepointSerializer { /** Random magic number for consistency checks */ private static final int MASTER_STATE_MAGIC_NUMBER = 0xc96b1696; @@ -320,7 +324,8 @@ private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) keyedStateStream); } - private static void serializeKeyedStateHandle( + @VisibleForTesting + public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { @@ -380,7 +385,8 @@ private static Map deserializeStreamStateHandl return result; } - private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { + @VisibleForTesting + public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { @@ -433,7 +439,8 @@ private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) } } - private static void serializeOperatorStateHandle( + @VisibleForTesting + public static void serializeOperatorStateHandle( OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle != null) { @@ -461,7 +468,8 @@ private static void serializeOperatorStateHandle( } } - private static OperatorStateHandle deserializeOperatorStateHandle( + @VisibleForTesting + public static OperatorStateHandle deserializeOperatorStateHandle( DataInputStream dis) throws IOException { final int type = dis.readByte(); @@ -492,7 +500,8 @@ private static OperatorStateHandle deserializeOperatorStateHandle( } } - private static void serializeStreamStateHandle( + @VisibleForTesting + public static void serializeStreamStateHandle( StreamStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { @@ -518,7 +527,7 @@ private static void serializeStreamStateHandle( dos.flush(); } - private static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException { + public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException { final int type = dis.read(); if (NULL_HANDLE == type) { return null; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java index 7392030496417..414cb471d7003 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java @@ -239,7 +239,7 @@ private static void retryOperationWithDelay( * @return Future which retries the given operation a given amount of times and delays the retry * in case the predicate isn't matched */ - public static CompletableFuture retrySuccesfulWithDelay( + public static CompletableFuture retrySuccessfulWithDelay( final Supplier> operation, final Time retryDelay, final Deadline deadline, @@ -518,9 +518,6 @@ private static class ResultConjunctFuture extends ConjunctFuture extends ConjunctFuture future : resultFutures) { - future.whenComplete(this::handleCompletedFuture); + final int index = counter; + counter++; + future.whenComplete((value, throwable) -> handleCompletedFuture(index, value, throwable)); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java index 4f5b2311a24a2..bb038eb80a6ac 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java @@ -208,10 +208,10 @@ public SerializedValue getSerializedJobInformation() { */ @Nullable public SerializedValue getSerializedTaskInformation() { - if (serializedJobInformation instanceof NonOffloaded) { - NonOffloaded jobInformation = + if (serializedTaskInformation instanceof NonOffloaded) { + NonOffloaded taskInformation = (NonOffloaded) serializedTaskInformation; - return jobInformation.serializedValue; + return taskInformation.serializedValue; } else { throw new IllegalStateException( "Trying to work with offloaded serialized job information."); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 3942d763ae476..48b7120f1c271 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.entrypoint; import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.ClusterOptions; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; @@ -53,10 +54,13 @@ import org.apache.flink.runtime.security.SecurityConfiguration; import org.apache.flink.runtime.security.SecurityContext; import org.apache.flink.runtime.security.SecurityUtils; +import org.apache.flink.runtime.util.ExecutorThreadFactory; +import org.apache.flink.runtime.util.Hardware; import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever; import org.apache.flink.util.AutoCloseableAsync; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.ExecutorUtils; import org.apache.flink.util.FileUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ShutdownHookUtil; @@ -80,6 +84,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -129,6 +135,9 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro @GuardedBy("lock") private RpcService commonRpcService; + @GuardedBy("lock") + private ExecutorService ioExecutor; + @GuardedBy("lock") private ActorSystem metricQueryServiceActorSystem; @@ -258,7 +267,10 @@ protected void initializeServices(Configuration configuration) throws Exception configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); - haServices = createHaServices(configuration, commonRpcService.getExecutor()); + ioExecutor = Executors.newFixedThreadPool( + Hardware.getNumberCPUCores(), + new ExecutorThreadFactory("cluster-io")); + haServices = createHaServices(configuration, ioExecutor); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); @@ -324,6 +336,8 @@ public CompletableFuture closeAsync() { } protected CompletableFuture stopClusterServices(boolean cleanupHaData) { + final long shutdownTimeout = configuration.getLong(ClusterOptions.CLUSTER_SERVICES_SHUTDOWN_TIMEOUT); + synchronized (lock) { Throwable exception = null; @@ -373,6 +387,10 @@ protected CompletableFuture stopClusterServices(boolean cleanupHaData) { terminationFutures.add(AkkaUtils.terminateActorSystem(metricQueryServiceActorSystem)); } + if (ioExecutor != null) { + terminationFutures.add(ExecutorUtils.nonBlockingShutdown(shutdownTimeout, TimeUnit.MILLISECONDS, ioExecutor)); + } + if (commonRpcService != null) { terminationFutures.add(commonRpcService.stopService()); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index cbf51037b8bb0..e3b501e52e837 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -41,6 +41,7 @@ import org.apache.flink.runtime.io.network.partition.ResultPartitionID; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationConstraint; import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint; +import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; @@ -427,7 +428,18 @@ public CompletableFuture scheduleForExecution( deploymentFuture.whenComplete( (Void ignored, Throwable failure) -> { if (failure != null) { - markFailed(ExceptionUtils.stripCompletionException(failure)); + final Throwable stripCompletionException = ExceptionUtils.stripCompletionException(failure); + final Throwable schedulingFailureCause; + + if (stripCompletionException instanceof TimeoutException) { + schedulingFailureCause = new NoResourceAvailableException( + "Could not allocate enough slots within timeout of " + allocationTimeout + " to run the job. " + + "Please make sure that the cluster has enough resources."); + } else { + schedulingFailureCause = stripCompletionException; + } + + markFailed(schedulingFailureCause); } }); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index 3b55e009116f5..56315e07146d5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -911,7 +911,7 @@ private CompletableFuture scheduleLazy(SlotProvider slotProvider) { final CompletableFuture schedulingJobVertexFuture = ejv.scheduleAll( slotProvider, allowQueuedScheduling, - LocationPreferenceConstraint.ALL,// since it is an input vertex, the input based location preferences should be empty + LocationPreferenceConstraint.ALL, // since it is an input vertex, the input based location preferences should be empty Collections.emptySet()); schedulingFutures.add(schedulingJobVertexFuture); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java index dafefcccef29f..3ca33fcbd62d7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.highavailability.nonha.embedded; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.runtime.leaderelection.LeaderContender; import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; @@ -96,6 +97,13 @@ public void shutdown() { } } + @VisibleForTesting + public boolean isShutdown() { + synchronized (lock) { + return shutdown; + } + } + private void fatalError(Throwable error) { LOG.error("Embedded leader election service encountered a fatal error. Shutting down service.", error); @@ -270,12 +278,13 @@ private void updateLeader() { currentLeaderSessionId = leaderSessionId; currentLeaderProposed = leaderService; + currentLeaderProposed.isLeader = true; LOG.info("Proposing leadership to contender {} @ {}", leaderService.contender, leaderService.contender.getAddress()); notificationExecutor.execute( - new GrantLeadershipCall(leaderService, leaderSessionId, LOG)); + new GrantLeadershipCall(leaderService.contender, leaderSessionId, LOG)); } } } @@ -440,34 +449,44 @@ public void run() { private static class GrantLeadershipCall implements Runnable { - private final EmbeddedLeaderElectionService leaderElectionService; + private final LeaderContender contender; private final UUID leaderSessionId; private final Logger logger; GrantLeadershipCall( - EmbeddedLeaderElectionService leaderElectionService, + LeaderContender contender, UUID leaderSessionId, Logger logger) { - this.leaderElectionService = checkNotNull(leaderElectionService); + this.contender = checkNotNull(contender); this.leaderSessionId = checkNotNull(leaderSessionId); this.logger = checkNotNull(logger); } @Override public void run() { - leaderElectionService.isLeader = true; - - final LeaderContender contender = leaderElectionService.contender; - try { contender.grantLeadership(leaderSessionId); } catch (Throwable t) { logger.warn("Error granting leadership to contender", t); contender.handleError(t instanceof Exception ? (Exception) t : new Exception(t)); - leaderElectionService.isLeader = false; } } } + + private static class RevokeLeadershipCall implements Runnable { + + @Nonnull + private final LeaderContender contender; + + RevokeLeadershipCall(@Nonnull LeaderContender contender) { + this.contender = contender; + } + + @Override + public void run() { + contender.revokeLeadership(); + } + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferListener.java index 4cc32c0a66147..e6b5416d98662 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferListener.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferListener.java @@ -24,6 +24,41 @@ */ public interface BufferListener { + /** + * Status of the notification result from the buffer listener. + */ + enum NotificationResult { + BUFFER_NOT_USED(false, false), + BUFFER_USED_NO_NEED_MORE(true, false), + BUFFER_USED_NEED_MORE(true, true); + + private final boolean isBufferUsed; + private final boolean needsMoreBuffers; + + NotificationResult(boolean isBufferUsed, boolean needsMoreBuffers) { + this.isBufferUsed = isBufferUsed; + this.needsMoreBuffers = needsMoreBuffers; + } + + /** + * Whether the notified buffer is accepted to use by the listener. + * + * @return true if the notified buffer is accepted. + */ + boolean isBufferUsed() { + return isBufferUsed; + } + + /** + * Whether the listener still needs more buffers to be notified. + * + * @return true if the listener is still waiting for more buffers. + */ + boolean needsMoreBuffers() { + return needsMoreBuffers; + } + } + /** * Notification callback if a buffer is recycled and becomes available in buffer pool. * @@ -37,9 +72,9 @@ public interface BufferListener { * stack! * * @param buffer buffer that becomes available in buffer pool. - * @return true if the listener wants to be notified next time. + * @return NotificationResult if the listener wants to be notified next time. */ - boolean notifyBufferAvailable(Buffer buffer); + NotificationResult notifyBufferAvailable(Buffer buffer); /** * Notification callback if the buffer provider is destroyed. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java index d7bfb603eafa9..dfad5b9cfaba0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.io.network.buffer; import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.runtime.io.network.buffer.BufferListener.NotificationResult; import org.apache.flink.util.ExceptionUtils; import org.slf4j.Logger; @@ -271,30 +272,31 @@ private MemorySegment requestMemorySegment(boolean isBlocking) throws Interrupte @Override public void recycle(MemorySegment segment) { BufferListener listener; - synchronized (availableMemorySegments) { - if (isDestroyed || numberOfRequestedMemorySegments > currentPoolSize) { - returnMemorySegment(segment); - return; - } else { - listener = registeredListeners.poll(); - - if (listener == null) { - availableMemorySegments.add(segment); - availableMemorySegments.notify(); + NotificationResult notificationResult = NotificationResult.BUFFER_NOT_USED; + while (!notificationResult.isBufferUsed()) { + synchronized (availableMemorySegments) { + if (isDestroyed || numberOfRequestedMemorySegments > currentPoolSize) { + returnMemorySegment(segment); return; + } else { + listener = registeredListeners.poll(); + if (listener == null) { + availableMemorySegments.add(segment); + availableMemorySegments.notify(); + return; + } } } + notificationResult = fireBufferAvailableNotification(listener, segment); } + } + private NotificationResult fireBufferAvailableNotification(BufferListener listener, MemorySegment segment) { // We do not know which locks have been acquired before the recycle() or are needed in the // notification and which other threads also access them. // -> call notifyBufferAvailable() outside of the synchronized block to avoid a deadlock (FLINK-9676) - // Note that in case of any exceptions notifyBufferAvailable() should recycle the buffer - // (either directly or later during error handling) and therefore eventually end up in this - // method again. - boolean needMoreBuffers = listener.notifyBufferAvailable(new NetworkBuffer(segment, this)); - - if (needMoreBuffers) { + NotificationResult notificationResult = listener.notifyBufferAvailable(new NetworkBuffer(segment, this)); + if (notificationResult.needsMoreBuffers()) { synchronized (availableMemorySegments) { if (isDestroyed) { // cleanup tasks how they would have been done if we only had one synchronized block @@ -304,6 +306,7 @@ public void recycle(MemorySegment segment) { } } } + return notificationResult; } /** @@ -404,5 +407,4 @@ private void returnExcessMemorySegments() { returnMemorySegment(segment); } } - } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestClientHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestClientHandler.java index c5ba7a4b7f1ac..34f65c0f22dc1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestClientHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestClientHandler.java @@ -417,32 +417,18 @@ public void notifyBufferDestroyed() { // Called by the recycling thread (not network I/O thread) @Override - public boolean notifyBufferAvailable(Buffer buffer) { - boolean success = false; - - try { - if (availableBuffer.compareAndSet(null, buffer)) { - ctx.channel().eventLoop().execute(this); + public NotificationResult notifyBufferAvailable(Buffer buffer) { + if (availableBuffer.compareAndSet(null, buffer)) { + ctx.channel().eventLoop().execute(this); - success = true; - } - else { - throw new IllegalStateException("Received a buffer notification, " + - " but the previous one has not been handled yet."); - } - } - catch (Throwable t) { - ctx.channel().eventLoop().execute(new AsyncErrorNotificationTask(t)); + return NotificationResult.BUFFER_USED_NO_NEED_MORE; } - finally { - if (!success) { - if (buffer != null) { - buffer.recycleBuffer(); - } - } + else { + ctx.channel().eventLoop().execute(new AsyncErrorNotificationTask( + new IllegalStateException("Received a buffer notification, " + + " but the previous one has not been handled yet."))); + return NotificationResult.BUFFER_NOT_USED; } - - return false; } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java index 6738abd7f9c05..141494996c792 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java @@ -342,18 +342,18 @@ boolean isWaitingForFloatingBuffers() { /** * The Buffer pool notifies this channel of an available floating buffer. If the channel is released or - * currently does not need extra buffers, the buffer should be recycled to the buffer pool. Otherwise, + * currently does not need extra buffers, the buffer should be returned to the buffer pool. Otherwise, * the buffer will be added into the bufferQueue and the unannounced credit is increased * by one. * * @param buffer Buffer that becomes available in buffer pool. - * @return True when this channel is waiting for more floating buffers, otherwise false. + * @return NotificationResult indicates whether this channel accepts the buffer and is waiting for + * more floating buffers. */ @Override - public boolean notifyBufferAvailable(Buffer buffer) { - boolean recycleBuffer = true; + public NotificationResult notifyBufferAvailable(Buffer buffer) { + NotificationResult notificationResult = NotificationResult.BUFFER_NOT_USED; try { - boolean needMoreBuffers = false; synchronized (bufferQueue) { checkState(isWaitingForFloatingBuffers, "This channel should be waiting for floating buffers."); @@ -364,36 +364,29 @@ public boolean notifyBufferAvailable(Buffer buffer) { // -> then isReleased is set correctly // 2) releaseAllResources() did not yet release buffers from bufferQueue // -> we may or may not have set isReleased yet but will always wait for the - // lock on bufferQueue to release buffers + // lock on bufferQueue to release buffers if (isReleased.get() || bufferQueue.getAvailableBufferSize() >= numRequiredBuffers) { isWaitingForFloatingBuffers = false; - recycleBuffer = false; // just in case - buffer.recycleBuffer(); - return false; + return notificationResult; } - recycleBuffer = false; bufferQueue.addFloatingBuffer(buffer); if (bufferQueue.getAvailableBufferSize() == numRequiredBuffers) { isWaitingForFloatingBuffers = false; + notificationResult = NotificationResult.BUFFER_USED_NO_NEED_MORE; } else { - needMoreBuffers = true; + notificationResult = NotificationResult.BUFFER_USED_NEED_MORE; } } if (unannouncedCredit.getAndAdd(1) == 0) { notifyCreditAvailable(); } - - return needMoreBuffers; } catch (Throwable t) { - if (recycleBuffer) { - buffer.recycleBuffer(); - } setError(t); - return false; } + return notificationResult; } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index 5d2d363cf71b6..2fdf79c7cb2fa 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -683,20 +683,18 @@ public void acknowledgeCheckpoint( } }); } else { - log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator", - jobGraph.getJobID()); + String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator"; + if (executionGraph.getState() == JobStatus.RUNNING) { + log.error(errorMessage, jobGraph.getJobID()); + } else { + log.debug(errorMessage, jobGraph.getJobID()); + } } } // TODO: This method needs a leader session ID @Override - public void declineCheckpoint( - final JobID jobID, - final ExecutionAttemptID executionAttemptID, - final long checkpointID, - final Throwable reason) { - final DeclineCheckpoint decline = new DeclineCheckpoint( - jobID, executionAttemptID, checkpointID, reason); + public void declineCheckpoint(DeclineCheckpoint decline) { final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); if (checkpointCoordinator != null) { @@ -708,8 +706,12 @@ public void declineCheckpoint( } }); } else { - log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator", - jobGraph.getJobID()); + String errorMessage = "Received DeclineCheckpoint message for job {} with no CheckpointCoordinator"; + if (executionGraph.getState() == JobStatus.RUNNING) { + log.error(errorMessage, jobGraph.getJobID()); + } else { + log.debug(errorMessage, jobGraph.getJobID()); + } } } @@ -962,19 +964,18 @@ public CompletableFuture triggerSavepoint( return checkpointCoordinator .triggerSavepoint(System.currentTimeMillis(), targetDirectory) .thenApply(CompletedCheckpoint::getExternalPointer) - .thenApplyAsync(path -> { - if (cancelJob) { + .handleAsync((path, throwable) -> { + if (throwable != null) { + if (cancelJob) { + startCheckpointScheduler(checkpointCoordinator); + } + throw new CompletionException(throwable); + } else if (cancelJob) { log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID()); cancel(timeout); } return path; - }, getMainThreadExecutor()) - .exceptionally(throwable -> { - if (cancelJob) { - startCheckpointScheduler(checkpointCoordinator); - } - throw new CompletionException(throwable); - }); + }, getMainThreadExecutor()); } private void startCheckpointScheduler(final CheckpointCoordinator checkpointCoordinator) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricNames.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricNames.java index dfa82f153816a..5953dc7353144 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricNames.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricNames.java @@ -57,4 +57,8 @@ private MetricNames() { public static final String TASK_SLOTS_AVAILABLE = "taskSlotsAvailable"; public static final String TASK_SLOTS_TOTAL = "taskSlotsTotal"; public static final String NUM_REGISTERED_TASK_MANAGERS = "numRegisteredTaskManagers"; + + public static final String MEMORY_USED = "Used"; + public static final String MEMORY_COMMITTED = "Committed"; + public static final String MEMORY_MAX = "Max"; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java index 7188a597c8642..244a1ede5ca20 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.metrics; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.AkkaOptions; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DelegatingConfiguration; @@ -26,6 +27,8 @@ import org.apache.flink.runtime.metrics.scope.ScopeFormats; import org.apache.flink.util.Preconditions; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,14 +69,18 @@ public class MetricRegistryConfiguration { // contains for every configured reporter its name and the configuration object private final List> reporterConfigurations; + private final long queryServiceMessageSizeLimit; + public MetricRegistryConfiguration( ScopeFormats scopeFormats, char delimiter, - List> reporterConfigurations) { + List> reporterConfigurations, + long queryServiceMessageSizeLimit) { this.scopeFormats = Preconditions.checkNotNull(scopeFormats); this.delimiter = delimiter; this.reporterConfigurations = Preconditions.checkNotNull(reporterConfigurations); + this.queryServiceMessageSizeLimit = queryServiceMessageSizeLimit; } // ------------------------------------------------------------------------ @@ -92,6 +99,10 @@ public List> getReporterConfigurations() { return reporterConfigurations; } + public long getQueryServiceMessageSizeLimit() { + return queryServiceMessageSizeLimit; + } + // ------------------------------------------------------------------------ // Static factory methods // ------------------------------------------------------------------------ @@ -160,7 +171,15 @@ public static MetricRegistryConfiguration fromConfiguration(Configuration config } } - return new MetricRegistryConfiguration(scopeFormats, delim, reporterConfigurations); + final String maxFrameSizeStr = configuration.getString(AkkaOptions.FRAMESIZE); + final String akkaConfigStr = String.format("akka {remote {netty.tcp {maximum-frame-size = %s}}}", maxFrameSizeStr); + final Config akkaConfig = ConfigFactory.parseString(akkaConfigStr); + final long maximumFrameSize = akkaConfig.getBytes("akka.remote.netty.tcp.maximum-frame-size"); + + // padding to account for serialization overhead + final long messageSizeLimitPadding = 256; + + return new MetricRegistryConfiguration(scopeFormats, delim, reporterConfigurations, maximumFrameSize - messageSizeLimitPadding); } public static MetricRegistryConfiguration defaultMetricRegistryConfiguration() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java index 6b3770907a94a..31775e242763f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java @@ -77,6 +77,8 @@ public class MetricRegistryImpl implements MetricRegistry { private final CompletableFuture terminationFuture; + private final long maximumFramesize; + @Nullable private ActorRef queryService; @@ -91,6 +93,7 @@ public class MetricRegistryImpl implements MetricRegistry { * Creates a new MetricRegistry and starts the configured reporter. */ public MetricRegistryImpl(MetricRegistryConfiguration config) { + this.maximumFramesize = config.getQueryServiceMessageSizeLimit(); this.scopeFormats = config.getScopeFormats(); this.globalDelimiter = config.getDelimiter(); this.delimiters = new ArrayList<>(10); @@ -184,7 +187,7 @@ public void startQueryService(ActorSystem actorSystem, ResourceID resourceID) { Preconditions.checkState(!isShutdown(), "The metric registry has already been shut down."); try { - queryService = MetricQueryService.startMetricQueryService(actorSystem, resourceID); + queryService = MetricQueryService.startMetricQueryService(actorSystem, resourceID, maximumFramesize); metricQueryServicePath = AkkaUtils.getAkkaURL(actorSystem, queryService); } catch (Exception e) { LOG.warn("Could not start MetricDumpActor. No metrics will be submitted to the WebInterface.", e); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java index 16a885dd345b6..5456b56cdbd07 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java @@ -73,19 +73,38 @@ public static class MetricSerializationResult implements Serializable { private static final long serialVersionUID = 6928770855951536906L; - public final byte[] serializedMetrics; + public final byte[] serializedCounters; + public final byte[] serializedGauges; + public final byte[] serializedMeters; + public final byte[] serializedHistograms; + public final int numCounters; public final int numGauges; public final int numMeters; public final int numHistograms; - public MetricSerializationResult(byte[] serializedMetrics, int numCounters, int numGauges, int numMeters, int numHistograms) { - Preconditions.checkNotNull(serializedMetrics); + public MetricSerializationResult( + byte[] serializedCounters, + byte[] serializedGauges, + byte[] serializedMeters, + byte[] serializedHistograms, + int numCounters, + int numGauges, + int numMeters, + int numHistograms) { + + Preconditions.checkNotNull(serializedCounters); + Preconditions.checkNotNull(serializedGauges); + Preconditions.checkNotNull(serializedMeters); + Preconditions.checkNotNull(serializedHistograms); Preconditions.checkArgument(numCounters >= 0); Preconditions.checkArgument(numGauges >= 0); Preconditions.checkArgument(numMeters >= 0); Preconditions.checkArgument(numHistograms >= 0); - this.serializedMetrics = serializedMetrics; + this.serializedCounters = serializedCounters; + this.serializedGauges = serializedGauges; + this.serializedMeters = serializedMeters; + this.serializedHistograms = serializedHistograms; this.numCounters = numCounters; this.numGauges = numGauges; this.numMeters = numMeters; @@ -102,7 +121,10 @@ public MetricSerializationResult(byte[] serializedMetrics, int numCounters, int */ public static class MetricDumpSerializer { - private DataOutputSerializer buffer = new DataOutputSerializer(1024 * 32); + private DataOutputSerializer countersBuffer = new DataOutputSerializer(1024 * 8); + private DataOutputSerializer gaugesBuffer = new DataOutputSerializer(1024 * 8); + private DataOutputSerializer metersBuffer = new DataOutputSerializer(1024 * 8); + private DataOutputSerializer histogramsBuffer = new DataOutputSerializer(1024 * 8); /** * Serializes the given metrics and returns the resulting byte array. @@ -126,53 +148,66 @@ public MetricSerializationResult serialize( Map> histograms, Map> meters) { - buffer.clear(); - + countersBuffer.clear(); int numCounters = 0; for (Map.Entry> entry : counters.entrySet()) { try { - serializeCounter(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); + serializeCounter(countersBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); numCounters++; } catch (Exception e) { LOG.debug("Failed to serialize counter.", e); } } + gaugesBuffer.clear(); int numGauges = 0; for (Map.Entry, Tuple2> entry : gauges.entrySet()) { try { - serializeGauge(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); + serializeGauge(gaugesBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); numGauges++; } catch (Exception e) { LOG.debug("Failed to serialize gauge.", e); } } + histogramsBuffer.clear(); int numHistograms = 0; for (Map.Entry> entry : histograms.entrySet()) { try { - serializeHistogram(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); + serializeHistogram(histogramsBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); numHistograms++; } catch (Exception e) { LOG.debug("Failed to serialize histogram.", e); } } + metersBuffer.clear(); int numMeters = 0; for (Map.Entry> entry : meters.entrySet()) { try { - serializeMeter(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); + serializeMeter(metersBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey()); numMeters++; } catch (Exception e) { LOG.debug("Failed to serialize meter.", e); } } - return new MetricSerializationResult(buffer.getCopyOfBuffer(), numCounters, numGauges, numMeters, numHistograms); + return new MetricSerializationResult( + countersBuffer.getCopyOfBuffer(), + gaugesBuffer.getCopyOfBuffer(), + metersBuffer.getCopyOfBuffer(), + histogramsBuffer.getCopyOfBuffer(), + numCounters, + numGauges, + numMeters, + numHistograms); } public void close() { - buffer = null; + countersBuffer = null; + gaugesBuffer = null; + metersBuffer = null; + histogramsBuffer = null; } } @@ -280,13 +315,16 @@ public static class MetricDumpDeserializer { * @return A list containing the deserialized metrics. */ public List deserialize(MetricDumpSerialization.MetricSerializationResult data) { - DataInputView in = new DataInputDeserializer(data.serializedMetrics, 0, data.serializedMetrics.length); + DataInputView countersInputView = new DataInputDeserializer(data.serializedCounters, 0, data.serializedCounters.length); + DataInputView gaugesInputView = new DataInputDeserializer(data.serializedGauges, 0, data.serializedGauges.length); + DataInputView metersInputView = new DataInputDeserializer(data.serializedMeters, 0, data.serializedMeters.length); + DataInputView histogramsInputView = new DataInputDeserializer(data.serializedHistograms, 0, data.serializedHistograms.length); - List metrics = new ArrayList<>(data.numCounters + data.numGauges + data.numHistograms + data.numMeters); + List metrics = new ArrayList<>(data.numCounters + data.numGauges + data.numMeters + data.numHistograms); for (int x = 0; x < data.numCounters; x++) { try { - metrics.add(deserializeCounter(in)); + metrics.add(deserializeCounter(countersInputView)); } catch (Exception e) { LOG.debug("Failed to deserialize counter.", e); } @@ -294,25 +332,25 @@ public List deserialize(MetricDumpSerialization.MetricSerializationR for (int x = 0; x < data.numGauges; x++) { try { - metrics.add(deserializeGauge(in)); + metrics.add(deserializeGauge(gaugesInputView)); } catch (Exception e) { LOG.debug("Failed to deserialize gauge.", e); } } - for (int x = 0; x < data.numHistograms; x++) { + for (int x = 0; x < data.numMeters; x++) { try { - metrics.add(deserializeHistogram(in)); + metrics.add(deserializeMeter(metersInputView)); } catch (Exception e) { - LOG.debug("Failed to deserialize histogram.", e); + LOG.debug("Failed to deserialize meter.", e); } } - for (int x = 0; x < data.numMeters; x++) { + for (int x = 0; x < data.numHistograms; x++) { try { - metrics.add(deserializeMeter(in)); + metrics.add(deserializeHistogram(histogramsInputView)); } catch (Exception e) { - LOG.debug("Failed to deserialize meter.", e); + LOG.debug("Failed to deserialize histogram.", e); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java index 8821e0d9f4a14..fc69d17503d05 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java @@ -55,6 +55,7 @@ public class MetricQueryService extends UntypedActor { private static final Logger LOG = LoggerFactory.getLogger(MetricQueryService.class); public static final String METRIC_QUERY_SERVICE_NAME = "MetricQueryService"; + private static final String SIZE_EXCEEDED_LOG_TEMPLATE = "{} will not be reported as the metric dump would exceed the maximum size of {} bytes."; private static final CharacterFilter FILTER = new CharacterFilter() { @Override @@ -70,6 +71,12 @@ public String filterCharacters(String input) { private final Map> histograms = new HashMap<>(); private final Map> meters = new HashMap<>(); + private final long messageSizeLimit; + + public MetricQueryService(long messageSizeLimit) { + this.messageSizeLimit = messageSizeLimit; + } + @Override public void postStop() { serializer.close(); @@ -109,6 +116,9 @@ public void onReceive(Object message) { } } else if (message instanceof CreateDump) { MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters); + + dump = enforceSizeLimit(dump); + getSender().tell(dump, getSelf()); } else { LOG.warn("MetricQueryServiceActor received an invalid message. " + message.toString()); @@ -119,6 +129,83 @@ public void onReceive(Object message) { } } + private MetricDumpSerialization.MetricSerializationResult enforceSizeLimit( + MetricDumpSerialization.MetricSerializationResult serializationResult) { + + int currentLength = 0; + boolean hasExceededBefore = false; + + byte[] serializedCounters = serializationResult.serializedCounters; + int numCounters = serializationResult.numCounters; + if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedCounters.length)) { + logDumpSizeWouldExceedLimit("Counters", hasExceededBefore); + hasExceededBefore = true; + + serializedCounters = new byte[0]; + numCounters = 0; + } else { + currentLength += serializedCounters.length; + } + + byte[] serializedMeters = serializationResult.serializedMeters; + int numMeters = serializationResult.numMeters; + if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedMeters.length)) { + logDumpSizeWouldExceedLimit("Meters", hasExceededBefore); + hasExceededBefore = true; + + serializedMeters = new byte[0]; + numMeters = 0; + } else { + currentLength += serializedMeters.length; + } + + byte[] serializedGauges = serializationResult.serializedGauges; + int numGauges = serializationResult.numGauges; + if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedGauges.length)) { + logDumpSizeWouldExceedLimit("Gauges", hasExceededBefore); + hasExceededBefore = true; + + serializedGauges = new byte[0]; + numGauges = 0; + } else { + currentLength += serializedGauges.length; + } + + byte[] serializedHistograms = serializationResult.serializedHistograms; + int numHistograms = serializationResult.numHistograms; + if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedHistograms.length)) { + logDumpSizeWouldExceedLimit("Histograms", hasExceededBefore); + hasExceededBefore = true; + + serializedHistograms = new byte[0]; + numHistograms = 0; + } + + return new MetricDumpSerialization.MetricSerializationResult( + serializedCounters, + serializedGauges, + serializedMeters, + serializedHistograms, + numCounters, + numGauges, + numMeters, + numHistograms); + } + + private boolean exceedsMessageSizeLimit(final int currentSize) { + return currentSize > messageSizeLimit; + } + + private void logDumpSizeWouldExceedLimit(final String metricType, boolean hasExceededBefore) { + if (LOG.isDebugEnabled()) { + LOG.debug(SIZE_EXCEEDED_LOG_TEMPLATE, metricType, messageSizeLimit); + } else { + if (!hasExceededBefore) { + LOG.info(SIZE_EXCEEDED_LOG_TEMPLATE, "Some metrics", messageSizeLimit); + } + } + } + /** * Lightweight method to replace unsupported characters. * If the string does not contain any unsupported characters, this method creates no @@ -165,11 +252,16 @@ static String replaceInvalidChars(String str) { * @param resourceID resource ID to disambiguate the actor name * @return actor reference to the MetricQueryService */ - public static ActorRef startMetricQueryService(ActorSystem actorSystem, ResourceID resourceID) { + public static ActorRef startMetricQueryService( + ActorSystem actorSystem, + ResourceID resourceID, + long maximumFramesize) { + String actorName = resourceID == null ? METRIC_QUERY_SERVICE_NAME : METRIC_QUERY_SERVICE_NAME + "_" + resourceID.getResourceIdString(); - return actorSystem.actorOf(Props.create(MetricQueryService.class), actorName); + + return actorSystem.actorOf(Props.create(MetricQueryService.class, maximumFramesize), actorName); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java index 909915f216e99..42e1cbbc0b71a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java @@ -90,9 +90,9 @@ public abstract class AbstractMetricGroup> impl * For example: "host-7.taskmanager-2.window_word_count.my-mapper" */ private final String[] scopeStrings; - /** The logical metrics scope represented by this group, as a concatenated string, lazily computed. + /** The logical metrics scope represented by this group for each reporter, as a concatenated string, lazily computed. * For example: "taskmanager.job.task" */ - private String logicalScopeString; + private String[] logicalScopeStrings; /** The metrics query service scope represented by this group, lazily computed. */ protected QueryScopeInfo queryServiceScopeInfo; @@ -107,20 +107,17 @@ public AbstractMetricGroup(MetricRegistry registry, String[] scope, A parent) { this.scopeComponents = checkNotNull(scope); this.parent = parent; this.scopeStrings = new String[registry.getNumberReporters()]; + this.logicalScopeStrings = new String[registry.getNumberReporters()]; } public Map getAllVariables() { - if (variables == null) { // avoid synchronization for common case - synchronized (this) { - if (variables == null) { - Map tmpVariables = new HashMap<>(); - putVariables(tmpVariables); - if (parent != null) { // not true for Job-/TaskManagerMetricGroup - tmpVariables.putAll(parent.getAllVariables()); - } - variables = tmpVariables; - } + if (variables == null) { + Map tmpVariables = new HashMap<>(); + putVariables(tmpVariables); + if (parent != null) { // not true for Job-/TaskManagerMetricGroup + tmpVariables.putAll(parent.getAllVariables()); } + variables = tmpVariables; } return variables; } @@ -152,14 +149,34 @@ public String getLogicalScope(CharacterFilter filter) { * @return logical scope */ public String getLogicalScope(CharacterFilter filter, char delimiter) { - if (logicalScopeString == null) { - if (parent == null) { - logicalScopeString = getGroupName(filter); - } else { - logicalScopeString = parent.getLogicalScope(filter, delimiter) + delimiter + getGroupName(filter); + return getLogicalScope(filter, delimiter, -1); + } + + /** + * Returns the logical scope of this group, for example + * {@code "taskmanager.job.task"}. + * + * @param filter character filter which is applied to the scope components + * @param delimiter delimiter to use for concatenating scope components + * @param reporterIndex index of the reporter + * @return logical scope + */ + String getLogicalScope(CharacterFilter filter, char delimiter, int reporterIndex) { + if (logicalScopeStrings.length == 0 || (reporterIndex < 0 || reporterIndex >= logicalScopeStrings.length)) { + return createLogicalScope(filter, delimiter); + } else { + if (logicalScopeStrings[reporterIndex] == null) { + logicalScopeStrings[reporterIndex] = createLogicalScope(filter, delimiter); } + return logicalScopeStrings[reporterIndex]; } - return logicalScopeString; + } + + protected String createLogicalScope(CharacterFilter filter, char delimiter) { + final String groupName = getGroupName(filter); + return parent == null + ? groupName + : parent.getLogicalScope(filter, delimiter) + delimiter + groupName; } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/FrontMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/FrontMetricGroup.java index 63842fef9d66e..64397d33db6bd 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/FrontMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/FrontMetricGroup.java @@ -52,6 +52,6 @@ public String getLogicalScope(CharacterFilter filter) { } public String getLogicalScope(CharacterFilter filter, char delimiter) { - return parentMetricGroup.getLogicalScope(filter, delimiter); + return parentMetricGroup.getLogicalScope(filter, delimiter, this.reporterIndex); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java index ef8e6e8bee8a2..41e9a91d38678 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java @@ -51,7 +51,7 @@ protected void putVariables(Map variables) { } @Override - public String getLogicalScope(CharacterFilter filter, char delimiter) { + protected String createLogicalScope(CharacterFilter filter, char delimiter) { return parent.getLogicalScope(filter, delimiter); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java index 6e6f49236b285..a0bbace5f8f38 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.metrics.util; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MetricOptions; @@ -26,6 +27,7 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools; import org.apache.flink.runtime.io.network.NetworkEnvironment; import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool; +import org.apache.flink.runtime.metrics.MetricNames; import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup; @@ -51,6 +53,7 @@ import java.lang.management.ThreadMXBean; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import static org.apache.flink.runtime.metrics.util.SystemResourcesMetricsInitializer.instantiateSystemMetrics; @@ -62,6 +65,9 @@ public class MetricUtils { private static final String METRIC_GROUP_STATUS_NAME = "Status"; private static final String METRICS_ACTOR_SYSTEM_NAME = "flink-metrics"; + static final String METRIC_GROUP_HEAP_NAME = "Heap"; + static final String METRIC_GROUP_NONHEAP_NAME = "NonHeap"; + private MetricUtils() { } @@ -159,20 +165,8 @@ private static void instantiateGarbageCollectorMetrics(MetricGroup metrics) { } private static void instantiateMemoryMetrics(MetricGroup metrics) { - final MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - final MemoryUsage nonHeapMemoryUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); - - MetricGroup heap = metrics.addGroup("Heap"); - - heap.>gauge("Used", heapMemoryUsage::getUsed); - heap.>gauge("Committed", heapMemoryUsage::getCommitted); - heap.>gauge("Max", heapMemoryUsage::getMax); - - MetricGroup nonHeap = metrics.addGroup("NonHeap"); - - nonHeap.>gauge("Used", nonHeapMemoryUsage::getUsed); - nonHeap.>gauge("Committed", nonHeapMemoryUsage::getCommitted); - nonHeap.>gauge("Max", nonHeapMemoryUsage::getMax); + instantiateHeapMemoryMetrics(metrics.addGroup(METRIC_GROUP_HEAP_NAME)); + instantiateNonHeapMemoryMetrics(metrics.addGroup(METRIC_GROUP_NONHEAP_NAME)); final MBeanServer con = ManagementFactory.getPlatformMBeanServer(); @@ -205,6 +199,22 @@ private static void instantiateMemoryMetrics(MetricGroup metrics) { } } + @VisibleForTesting + static void instantiateHeapMemoryMetrics(final MetricGroup metricGroup) { + instantiateMemoryUsageMetrics(metricGroup, () -> ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()); + } + + @VisibleForTesting + static void instantiateNonHeapMemoryMetrics(final MetricGroup metricGroup) { + instantiateMemoryUsageMetrics(metricGroup, () -> ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage()); + } + + private static void instantiateMemoryUsageMetrics(final MetricGroup metricGroup, final Supplier memoryUsageSupplier) { + metricGroup.>gauge(MetricNames.MEMORY_USED, () -> memoryUsageSupplier.get().getUsed()); + metricGroup.>gauge(MetricNames.MEMORY_COMMITTED, () -> memoryUsageSupplier.get().getCommitted()); + metricGroup.>gauge(MetricNames.MEMORY_MAX, () -> memoryUsageSupplier.get().getMax()); + } + private static void instantiateThreadMetrics(MetricGroup metrics) { final ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 69bad47e15813..f272361b2520b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -23,6 +23,7 @@ import org.apache.flink.api.common.JobSubmissionResult; import org.apache.flink.api.common.io.FileOutputFormat; import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.ClusterOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.RestOptions; @@ -71,11 +72,14 @@ import org.apache.flink.runtime.rpc.akka.AkkaRpcService; import org.apache.flink.runtime.taskexecutor.TaskExecutor; import org.apache.flink.runtime.taskexecutor.TaskManagerRunner; +import org.apache.flink.runtime.util.ExecutorThreadFactory; +import org.apache.flink.runtime.util.Hardware; import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever; import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever; import org.apache.flink.util.AutoCloseableAsync; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.ExecutorUtils; import org.apache.flink.util.FlinkException; import akka.actor.ActorSystem; @@ -97,6 +101,9 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -134,6 +141,9 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync { @GuardedBy("lock") private RpcService resourceManagerRpcService; + @GuardedBy("lock") + private ExecutorService ioExecutor; + @GuardedBy("lock") private ActorSystem metricQueryServiceActorSystem; @@ -188,7 +198,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync { public MiniCluster(MiniClusterConfiguration miniClusterConfiguration) { this.miniClusterConfiguration = checkNotNull(miniClusterConfiguration, "config may not be null"); - this.rpcTimeout = Time.seconds(10L); + this.rpcTimeout = miniClusterConfiguration.getRpcTimeout(); this.terminationFuture = CompletableFuture.completedFuture(null); running = false; } @@ -296,9 +306,12 @@ public void start() throws Exception { // create the high-availability services LOG.info("Starting high-availability services"); + ioExecutor = Executors.newFixedThreadPool( + Hardware.getNumberCPUCores(), + new ExecutorThreadFactory("mini-cluster-io")); haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices( configuration, - commonRpcService.getExecutor()); + ioExecutor); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); @@ -432,6 +445,7 @@ public CompletableFuture closeAsync() { if (running) { LOG.info("Shutting down Flink Mini Cluster"); try { + final long shutdownTimeoutMillis = miniClusterConfiguration.getConfiguration().getLong(ClusterOptions.CLUSTER_SERVICES_SHUTDOWN_TIMEOUT); final int numComponents = 2 + miniClusterConfiguration.getNumTaskManagers(); final Collection> componentTerminationFutures = new ArrayList<>(numComponents); @@ -466,7 +480,11 @@ public CompletableFuture closeAsync() { rpcServicesTerminationFuture, this::terminateMiniClusterServices); - remainingServicesTerminationFuture.whenComplete( + final CompletableFuture executorsTerminationFuture = FutureUtils.runAfterwards( + remainingServicesTerminationFuture, + () -> terminateExecutors(shutdownTimeoutMillis)); + + executorsTerminationFuture.whenComplete( (Void ignored, Throwable throwable) -> { if (throwable != null) { terminationFuture.completeExceptionally(ExceptionUtils.stripCompletionException(throwable)); @@ -946,6 +964,16 @@ private CompletionStage terminateRpcServices() { return FutureUtils.completeAll(rpcTerminationFutures); } + private CompletableFuture terminateExecutors(long executorShutdownTimeoutMillis) { + synchronized (lock) { + if (ioExecutor != null) { + return ExecutorUtils.nonBlockingShutdown(executorShutdownTimeoutMillis, TimeUnit.MILLISECONDS, ioExecutor); + } else { + return CompletableFuture.completedFuture(null); + } + } + } + // ------------------------------------------------------------------------ // miscellaneous utilities // ------------------------------------------------------------------------ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java index cd4281e11cfa0..f90f63a2d9163 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java @@ -1145,20 +1145,15 @@ public void run() { @Override public void reportPayload(final ResourceID resourceID, final SlotReport slotReport) { - runAsync(new Runnable() { - @Override - public void run() { - log.debug("Received new slot report from TaskManager {}.", resourceID); - - final WorkerRegistration workerRegistration = taskExecutors.get(resourceID); + runAsync(() -> { + final WorkerRegistration workerRegistration = taskExecutors.get(resourceID); - if (workerRegistration == null) { - log.debug("Received slot report from TaskManager {} which is no longer registered.", resourceID); - } else { - InstanceID instanceId = workerRegistration.getInstanceID(); + if (workerRegistration == null) { + log.debug("Received slot report from TaskManager {} which is no longer registered.", resourceID); + } else { + InstanceID instanceId = workerRegistration.getInstanceID(); - slotManager.reportSlotStatus(instanceId, slotReport); - } + slotManager.reportSlotStatus(instanceId, slotReport); } }); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java index 2ef2b2fefcc36..1dd16934280a5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java @@ -402,7 +402,7 @@ public boolean unregisterTaskManager(InstanceID instanceId) { public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) { checkInit(); - LOG.debug("Received slot report from instance {}.", instanceId); + LOG.debug("Received slot report from instance {}: {}.", instanceId, slotReport); TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java index 7c46af04b552c..b99de66122c09 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java @@ -102,6 +102,10 @@ protected void channelRead0(final ChannelHandlerContext ctx, final HttpObject ms checkState(currentUploadDir == null); currentHttpPostRequestDecoder = new HttpPostRequestDecoder(DATA_FACTORY, httpRequest); currentHttpRequest = ReferenceCountUtil.retain(httpRequest); + + // make sure that we still have a upload dir in case that it got deleted in the meanwhile + RestServerEndpoint.createUploadDir(uploadDir, LOG); + currentUploadDir = Files.createDirectory(uploadDir.resolve(UUID.randomUUID().toString())); } else { ctx.fireChannelRead(ReferenceCountUtil.retain(msg)); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java index 5a1c371d5a671..dc36aec47597e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java @@ -31,6 +31,7 @@ import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.AutoCloseableAsync; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParseException; @@ -50,6 +51,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -117,15 +119,13 @@ protected void respondAsLeader(ChannelHandlerContext ctx, RoutedRequest routedRe try { request = MAPPER.readValue("{}", untypedResponseMessageHeaders.getRequestClass()); } catch (JsonParseException | JsonMappingException je) { - log.error("Request did not conform to expected format.", je); - throw new RestHandlerException("Bad request received.", HttpResponseStatus.BAD_REQUEST, je); + throw new RestHandlerException("Bad request received. Request did not conform to expected format.", HttpResponseStatus.BAD_REQUEST, je); } } else { try { ByteBufInputStream in = new ByteBufInputStream(msgContent); request = MAPPER.readValue(in, untypedResponseMessageHeaders.getRequestClass()); } catch (JsonParseException | JsonMappingException je) { - log.error("Failed to read request.", je); throw new RestHandlerException( String.format("Request did not match expected format %s.", untypedResponseMessageHeaders.getRequestClass().getSimpleName()), HttpResponseStatus.BAD_REQUEST, @@ -160,28 +160,49 @@ protected void respondAsLeader(ChannelHandlerContext ctx, RoutedRequest routedRe final FileUploads finalUploadedFiles = uploadedFiles; requestProcessingFuture .whenComplete((Void ignored, Throwable throwable) -> { - inFlightRequestTracker.deregisterRequest(); - cleanupFileUploads(finalUploadedFiles); + if (throwable != null) { + handleException(ExceptionUtils.stripCompletionException(throwable), ctx, httpRequest) + .whenComplete((Void ignored2, Throwable throwable2) -> finalizeRequestProcessing(finalUploadedFiles)); + } else { + finalizeRequestProcessing(finalUploadedFiles); + } }); - } catch (RestHandlerException rhe) { - inFlightRequestTracker.deregisterRequest(); - HandlerUtils.sendErrorResponse( + } catch (Throwable e) { + final FileUploads finalUploadedFiles = uploadedFiles; + handleException(e, ctx, httpRequest) + .whenComplete((Void ignored, Throwable throwable) -> finalizeRequestProcessing(finalUploadedFiles)); + } + } + + private void finalizeRequestProcessing(FileUploads uploadedFiles) { + inFlightRequestTracker.deregisterRequest(); + cleanupFileUploads(uploadedFiles); + } + + private CompletableFuture handleException(Throwable throwable, ChannelHandlerContext ctx, HttpRequest httpRequest) { + if (throwable instanceof RestHandlerException) { + RestHandlerException rhe = (RestHandlerException) throwable; + if (log.isDebugEnabled()) { + log.error("Exception occurred in REST handler.", rhe); + } else { + log.error("Exception occurred in REST handler: {}", rhe.getMessage()); + } + return HandlerUtils.sendErrorResponse( ctx, httpRequest, new ErrorResponseBody(rhe.getMessage()), rhe.getHttpResponseStatus(), responseHeaders); - cleanupFileUploads(uploadedFiles); - } catch (Throwable e) { - inFlightRequestTracker.deregisterRequest(); - log.error("Request processing failed.", e); - HandlerUtils.sendErrorResponse( + } else { + log.error("Unhandled exception.", throwable); + String stackTrace = String.format("", + ExceptionUtils.stringifyException(throwable)); + return HandlerUtils.sendErrorResponse( ctx, httpRequest, - new ErrorResponseBody("Internal server error."), + new ErrorResponseBody(Arrays.asList("Internal server error.", stackTrace)), HttpResponseStatus.INTERNAL_SERVER_ERROR, responseHeaders); - cleanupFileUploads(uploadedFiles); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java index 0397cb875f295..992e2c58d061c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java @@ -19,17 +19,14 @@ package org.apache.flink.runtime.rest.handler; import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.rest.handler.util.HandlerUtils; -import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.MessageHeaders; import org.apache.flink.runtime.rest.messages.MessageParameters; import org.apache.flink.runtime.rest.messages.RequestBody; import org.apache.flink.runtime.rest.messages.ResponseBody; import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; -import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler; @@ -39,7 +36,6 @@ import javax.annotation.Nonnull; -import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -80,25 +76,7 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht response = FutureUtils.completedExceptionally(e); } - return response.handle((resp, throwable) -> throwable != null ? - errorResponse(throwable) : Tuple2.of(resp, messageHeaders.getResponseStatusCode())) - .thenCompose(r -> HandlerUtils.sendResponse(ctx, httpRequest, r.f0, r.f1, responseHeaders)); - } - - private Tuple2 errorResponse(Throwable throwable) { - Throwable error = ExceptionUtils.stripCompletionException(throwable); - if (error instanceof RestHandlerException) { - final RestHandlerException rhe = (RestHandlerException) error; - log.error("Exception occurred in REST handler.", rhe); - return Tuple2.of(new ErrorResponseBody(rhe.getMessage()), rhe.getHttpResponseStatus()); - } else { - log.error("Implementation error: Unhandled exception.", error); - String stackTrace = String.format("", - ExceptionUtils.stringifyException(throwable)); - return Tuple2.of( - new ErrorResponseBody(Arrays.asList("Internal server error.", stackTrace)), - HttpResponseStatus.INTERNAL_SERVER_ERROR); - } + return response.thenAccept(resp -> HandlerUtils.sendResponse(ctx, httpRequest, resp, messageHeaders.getResponseStatusCode(), responseHeaders)); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java index 8a20868ce3733..01d818bfaca9d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java @@ -24,12 +24,11 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.exceptions.UnknownTaskExecutorException; +import org.apache.flink.runtime.rest.NotFoundException; import org.apache.flink.runtime.rest.handler.AbstractHandler; import org.apache.flink.runtime.rest.handler.HandlerRequest; import org.apache.flink.runtime.rest.handler.RestHandlerException; -import org.apache.flink.runtime.rest.handler.util.HandlerUtils; import org.apache.flink.runtime.rest.messages.EmptyRequestBody; -import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.UntypedResponseMessageHeaders; import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerIdPathParameter; import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerMessageParameters; @@ -75,7 +74,6 @@ import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; -import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus.OK; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -126,11 +124,7 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht blobKeyFuture = fileBlobKeys.get(taskManagerId); } catch (ExecutionException e) { final Throwable cause = ExceptionUtils.stripExecutionException(e); - if (cause instanceof RestHandlerException) { - throw (RestHandlerException) cause; - } else { - throw new RestHandlerException("Could not retrieve file blob key future.", HttpResponseStatus.INTERNAL_SERVER_ERROR, e); - } + throw new RestHandlerException("Could not retrieve file blob key future.", HttpResponseStatus.INTERNAL_SERVER_ERROR, cause); } final CompletableFuture resultFuture = blobKeyFuture.thenAcceptAsync( @@ -160,23 +154,18 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht fileBlobKeys.invalidate(taskManagerId); final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable); - final ErrorResponseBody errorResponseBody; - final HttpResponseStatus httpResponseStatus; if (strippedThrowable instanceof UnknownTaskExecutorException) { - errorResponseBody = new ErrorResponseBody("Unknown TaskExecutor " + taskManagerId + '.'); - httpResponseStatus = HttpResponseStatus.NOT_FOUND; + throw new CompletionException( + new NotFoundException( + String.format("Failed to transfer file from TaskExecutor %s because it was unknown.", taskManagerId), + strippedThrowable)); } else { - errorResponseBody = new ErrorResponseBody("Internal server error: " + throwable.getMessage() + '.'); - httpResponseStatus = INTERNAL_SERVER_ERROR; + throw new CompletionException( + new FlinkException( + String.format("Failed to transfer file from TaskExecutor %s.", taskManagerId), + strippedThrowable)); } - - HandlerUtils.sendErrorResponse( - ctx, - httpRequest, - errorResponseBody, - httpResponseStatus, - responseHeaders); } }); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java index df007f45c0759..3ad1e411d29c8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java @@ -110,7 +110,8 @@ public static final class VertexTaskDetail { public static final String FIELD_NAME_STATUS = "status"; public static final String FIELD_NAME_ATTEMPT = "attempt"; public static final String FIELD_NAME_HOST = "host"; - public static final String FIELD_NAME_START_TIME = "start_time"; + public static final String FIELD_NAME_START_TIME = "start-time"; + public static final String FIELD_NAME_COMPATIBLE_START_TIME = "start_time"; public static final String FIELD_NAME_END_TIME = "end-time"; public static final String FIELD_NAME_DURATION = "duration"; public static final String FIELD_NAME_METRICS = "metrics"; @@ -130,6 +131,9 @@ public static final class VertexTaskDetail { @JsonProperty(FIELD_NAME_START_TIME) private final long startTime; + @JsonProperty(FIELD_NAME_COMPATIBLE_START_TIME) + private final long startTimeCompatible; + @JsonProperty(FIELD_NAME_END_TIME) private final long endTime; @@ -154,6 +158,7 @@ public VertexTaskDetail( this.attempt = attempt; this.host = checkNotNull(host); this.startTime = startTime; + this.startTimeCompatible = startTime; this.endTime = endTime; this.duration = duration; this.metrics = checkNotNull(metrics); @@ -175,6 +180,7 @@ public boolean equals(Object o) { attempt == that.attempt && Objects.equals(host, that.host) && startTime == that.startTime && + startTimeCompatible == that.startTimeCompatible && endTime == that.endTime && duration == that.duration && Objects.equals(metrics, that.metrics); @@ -182,7 +188,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(subtask, status, attempt, host, startTime, endTime, duration, metrics); + return Objects.hash(subtask, status, attempt, host, startTime, startTimeCompatible, endTime, duration, metrics); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/JobManagerMetricsMessageParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/JobManagerMetricsMessageParameters.java index 7fd63726e0e21..311a30034c80c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/JobManagerMetricsMessageParameters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/JobManagerMetricsMessageParameters.java @@ -31,7 +31,7 @@ */ public class JobManagerMetricsMessageParameters extends MessageParameters { - private final MetricsFilterParameter metricsFilterParameter = new MetricsFilterParameter(); + public final MetricsFilterParameter metricsFilterParameter = new MetricsFilterParameter(); @Override public Collection> getPathParameters() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/TaskManagerMetricsMessageParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/TaskManagerMetricsMessageParameters.java index d7e9381d18525..b30e4b8b3ec5d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/TaskManagerMetricsMessageParameters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/TaskManagerMetricsMessageParameters.java @@ -31,7 +31,7 @@ */ public class TaskManagerMetricsMessageParameters extends TaskManagerMessageParameters { - private final MetricsFilterParameter metricsFilterParameter = new MetricsFilterParameter(); + public final MetricsFilterParameter metricsFilterParameter = new MetricsFilterParameter(); @Override public Collection> getQueryParameters() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerMessageParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerMessageParameters.java index 59408a7304e5e..4a761215bc102 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerMessageParameters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerMessageParameters.java @@ -31,7 +31,7 @@ */ public class TaskManagerMessageParameters extends MessageParameters { - private TaskManagerIdPathParameter taskManagerIdParameter = new TaskManagerIdPathParameter(); + public TaskManagerIdPathParameter taskManagerIdParameter = new TaskManagerIdPathParameter(); @Override public Collection> getPathParameters() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcUtils.java index c90a8b5bbbc6e..2f656d092636b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcUtils.java @@ -19,9 +19,13 @@ package org.apache.flink.runtime.rpc; import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.concurrent.FutureUtils; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -87,6 +91,29 @@ public static void terminateRpcService(RpcService rpcService, Time timeout) thro rpcService.stopService().get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); } + /** + * Shuts the given rpc services down and waits for their termination. + * + * @param rpcServices to shut down + * @param timeout for this operation + * @throws InterruptedException if the operation has been interrupted + * @throws ExecutionException if a problem occurred + * @throws TimeoutException if a timeout occurred + */ + public static void terminateRpcServices( + Time timeout, + RpcService... rpcServices) throws InterruptedException, ExecutionException, TimeoutException { + final Collection> terminationFutures = new ArrayList<>(rpcServices.length); + + for (RpcService service : rpcServices) { + if (service != null) { + terminationFutures.add(service.stopService()); + } + } + + FutureUtils.waitForAll(terminationFutures).get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); + } + // We don't want this class to be instantiable private RpcUtils() {} } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/messages/RemoteRpcInvocation.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/messages/RemoteRpcInvocation.java index 7b9fb88767043..486816de8e80d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/messages/RemoteRpcInvocation.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/messages/RemoteRpcInvocation.java @@ -203,12 +203,18 @@ private void readObject(ObjectInputStream ois) throws IOException, ClassNotFound try { parameterTypes[i] = (Class) ois.readObject(); } catch (IOException e) { + StringBuilder incompleteMethod = getIncompleteMethodString(i, 0); throw new IOException("Could not deserialize " + i + "th parameter type of method " + - methodName + '.', e); + incompleteMethod + '.', e); } catch (ClassNotFoundException e) { - throw new ClassNotFoundException("Could not deserialize " + i + "th " + - "parameter type of method " + methodName + ". This indicates that the parameter " + - "type is not part of the system class loader.", e); + // note: wrapping this CNFE into another CNFE does not overwrite the Exception + // stored in the ObjectInputStream (see ObjectInputStream#readSerialData) + // -> add a suppressed exception that adds a more specific message + StringBuilder incompleteMethod = getIncompleteMethodString(i, 0); + e.addSuppressed(new ClassNotFoundException("Could not deserialize " + i + "th " + + "parameter type of method " + incompleteMethod + ". This indicates that the parameter " + + "type is not part of the system class loader.")); + throw e; } } @@ -221,17 +227,37 @@ private void readObject(ObjectInputStream ois) throws IOException, ClassNotFound try { args[i] = ois.readObject(); } catch (IOException e) { + StringBuilder incompleteMethod = getIncompleteMethodString(length, i); throw new IOException("Could not deserialize " + i + "th argument of method " + - methodName + '.', e); + incompleteMethod + '.', e); } catch (ClassNotFoundException e) { - throw new ClassNotFoundException("Could not deserialize " + i + "th " + - "argument of method " + methodName + ". This indicates that the argument " + - "type is not part of the system class loader.", e); + // note: wrapping this CNFE into another CNFE does not overwrite the Exception + // stored in the ObjectInputStream (see ObjectInputStream#readSerialData) + // -> add a suppressed exception that adds a more specific message + StringBuilder incompleteMethod = getIncompleteMethodString(length, i); + e.addSuppressed(new ClassNotFoundException("Could not deserialize " + i + "th " + + "argument of method " + incompleteMethod + ". This indicates that the argument " + + "type is not part of the system class loader.")); + throw e; } } } else { args = null; } } + + private StringBuilder getIncompleteMethodString(int lastMethodTypeIdx, int lastArgumentIdx) { + StringBuilder incompleteMethod = new StringBuilder(); + incompleteMethod.append(methodName).append('('); + for (int i = 0; i < lastMethodTypeIdx; ++i) { + incompleteMethod.append(parameterTypes[i].getCanonicalName()); + if (i < lastArgumentIdx) { + incompleteMethod.append(": ").append(args[i]); + } + incompleteMethod.append(", "); + } + incompleteMethod.append("...)"); // some parameters could not be deserialized + return incompleteMethod; + } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java index 4702919384fad..952dffbc70f19 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java @@ -26,7 +26,6 @@ import org.apache.flink.api.common.state.MapStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.core.fs.CloseableRegistry; import org.apache.flink.core.fs.FSDataInputStream; @@ -105,23 +104,10 @@ public class DefaultOperatorStateBackend implements OperatorStateBackend { */ private final boolean asynchronousSnapshots; - /** - * Map of state names to their corresponding restored state meta info. - * - *

    TODO this map can be removed when eager-state registration is in place. - * TODO we currently need this cached to check state migration strategies when new serializers are registered. - */ - private final Map restoredOperatorStateMetaInfos; - - /** - * Map of state names to their corresponding restored broadcast state meta info. - */ - private final Map restoredBroadcastStateMetaInfos; - /** * Cache of already accessed states. * - *

    In contrast to {@link #registeredOperatorStates} and {@link #restoredOperatorStateMetaInfos} which may be repopulated + *

    In contrast to {@link #registeredOperatorStates} which may be repopulated * with restored state, this map is always empty at the beginning. * *

    TODO this map should be moved to a base class once we have proper hierarchy for the operator state backends. @@ -148,8 +134,6 @@ public DefaultOperatorStateBackend( this.asynchronousSnapshots = asynchronousSnapshots; this.accessedStatesByName = new HashMap<>(); this.accessedBroadcastStatesByName = new HashMap<>(); - this.restoredOperatorStateMetaInfos = new HashMap<>(); - this.restoredBroadcastStateMetaInfos = new HashMap<>(); this.snapshotStrategy = new DefaultOperatorStateBackendSnapshotStrategy(); } @@ -226,34 +210,22 @@ public BroadcastState getBroadcastState(final MapStateDescriptor restoredBroadcastStateMetaInfo = broadcastState.getStateMetaInfo(); // check whether new serializers are incompatible - TypeSerializerSnapshot keySerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) metaInfoSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER)); - TypeSerializerSchemaCompatibility keyCompatibility = - keySerializerSnapshot.resolveSchemaCompatibility(broadcastStateKeySerializer); + restoredBroadcastStateMetaInfo.updateKeySerializer(broadcastStateKeySerializer); if (keyCompatibility.isIncompatible()) { throw new StateMigrationException("The new key serializer for broadcast state must not be incompatible."); } - TypeSerializerSnapshot valueSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) metaInfoSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)); - TypeSerializerSchemaCompatibility valueCompatibility = - valueSerializerSnapshot.resolveSchemaCompatibility(broadcastStateValueSerializer); + restoredBroadcastStateMetaInfo.updateValueSerializer(broadcastStateValueSerializer); if (valueCompatibility.isIncompatible()) { throw new StateMigrationException("The new value serializer for broadcast state must not be incompatible."); } - // new serializer is compatible; use it to replace the old serializer - broadcastState.setStateMetaInfo( - new RegisteredBroadcastStateBackendMetaInfo<>( - name, - OperatorStateHandle.Mode.BROADCAST, - broadcastStateKeySerializer, - broadcastStateValueSerializer)); + broadcastState.setStateMetaInfo(restoredBroadcastStateMetaInfo); } accessedBroadcastStatesByName.put(name, broadcastState); @@ -345,8 +317,6 @@ public void restore(Collection restoreSnapshots) throws Exc " not be loaded. This is a temporary restriction that will be fixed in future versions."); } - restoredOperatorStateMetaInfos.put(restoredSnapshot.getName(), restoredSnapshot); - PartitionableListState listState = registeredOperatorStates.get(restoredSnapshot.getName()); if (null == listState) { @@ -381,8 +351,6 @@ public void restore(Collection restoreSnapshots) throws Exc " not be loaded. This is a temporary restriction that will be fixed in future versions."); } - restoredBroadcastStateMetaInfos.put(restoredSnapshot.getName(), restoredSnapshot); - BackendWritableBroadcastState broadcastState = registeredBroadcastStates.get(restoredSnapshot.getName()); if (broadcastState == null) { @@ -590,25 +558,19 @@ private ListState getListState( partitionableListState.getStateMetaInfo().getAssignmentMode(), mode); - StateMetaInfoSnapshot restoredSnapshot = restoredOperatorStateMetaInfos.get(name); - RegisteredOperatorStateBackendMetaInfo metaInfo = - new RegisteredOperatorStateBackendMetaInfo<>(restoredSnapshot); + RegisteredOperatorStateBackendMetaInfo restoredPartitionableListStateMetaInfo = + partitionableListState.getStateMetaInfo(); - // check compatibility to determine if state migration is required + // check compatibility to determine if new serializers are incompatible TypeSerializer newPartitionStateSerializer = partitionStateSerializer.duplicate(); - @SuppressWarnings("unchecked") - TypeSerializerSnapshot stateSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)); - TypeSerializerSchemaCompatibility stateCompatibility = - stateSerializerSnapshot.resolveSchemaCompatibility(newPartitionStateSerializer); + restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer); if (stateCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer for operator state must not be incompatible."); } - partitionableListState.setStateMetaInfo( - new RegisteredOperatorStateBackendMetaInfo<>(name, newPartitionStateSerializer, mode)); + partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo); } accessedStatesByName.put(name, partitionableListState); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java index 70a14142474c2..ecc13faa43d22 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java @@ -19,11 +19,13 @@ package org.apache.flink.runtime.state; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot; import org.apache.flink.util.Preconditions; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Collections; import java.util.HashMap; @@ -38,11 +40,11 @@ public class RegisteredBroadcastStateBackendMetaInfo extends RegisteredSta /** The type serializer for the keys in the map state. */ @Nonnull - private final TypeSerializer keySerializer; + private final StateSerializerProvider keySerializerProvider; /** The type serializer for the values in the map state. */ @Nonnull - private final TypeSerializer valueSerializer; + private final StateSerializerProvider valueSerializerProvider; public RegisteredBroadcastStateBackendMetaInfo( @Nonnull final String name, @@ -50,19 +52,19 @@ public RegisteredBroadcastStateBackendMetaInfo( @Nonnull final TypeSerializer keySerializer, @Nonnull final TypeSerializer valueSerializer) { - super(name); - Preconditions.checkArgument(assignmentMode == OperatorStateHandle.Mode.BROADCAST); - this.assignmentMode = assignmentMode; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; + this( + name, + assignmentMode, + StateSerializerProvider.fromNewState(keySerializer), + StateSerializerProvider.fromNewState(valueSerializer)); } public RegisteredBroadcastStateBackendMetaInfo(@Nonnull RegisteredBroadcastStateBackendMetaInfo copy) { this( Preconditions.checkNotNull(copy).name, copy.assignmentMode, - copy.keySerializer.duplicate(), - copy.valueSerializer.duplicate()); + copy.getKeySerializer().duplicate(), + copy.getValueSerializer().duplicate()); } @SuppressWarnings("unchecked") @@ -71,10 +73,13 @@ public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sn snapshot.getName(), OperatorStateHandle.Mode.valueOf( snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER)), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))); + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER))), + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)))); + Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.BROADCAST == snapshot.getBackendStateType()); } @@ -86,6 +91,19 @@ public RegisteredBroadcastStateBackendMetaInfo deepCopy() { return new RegisteredBroadcastStateBackendMetaInfo<>(this); } + private RegisteredBroadcastStateBackendMetaInfo( + @Nonnull final String name, + @Nonnull final OperatorStateHandle.Mode assignmentMode, + @Nonnull final StateSerializerProvider keySerializerProvider, + @Nonnull final StateSerializerProvider valueSerializerProvider) { + + super(name); + Preconditions.checkArgument(assignmentMode == OperatorStateHandle.Mode.BROADCAST); + this.assignmentMode = assignmentMode; + this.keySerializerProvider = keySerializerProvider; + this.valueSerializerProvider = valueSerializerProvider; + } + @Nonnull @Override public StateMetaInfoSnapshot snapshot() { @@ -94,12 +112,32 @@ public StateMetaInfoSnapshot snapshot() { @Nonnull public TypeSerializer getKeySerializer() { - return keySerializer; + return keySerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updateKeySerializer(TypeSerializer newKeySerializer) { + return keySerializerProvider.registerNewSerializerForRestoredState(newKeySerializer); + } + + @Nullable + public TypeSerializer getPreviousKeySerializer() { + return keySerializerProvider.previousSchemaSerializer(); } @Nonnull public TypeSerializer getValueSerializer() { - return valueSerializer; + return valueSerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updateValueSerializer(TypeSerializer newValueSerializer) { + return valueSerializerProvider.registerNewSerializerForRestoredState(newValueSerializer); + } + + @Nullable + public TypeSerializer getPreviousValueSerializer() { + return valueSerializerProvider.previousSchemaSerializer(); } @Nonnull @@ -122,16 +160,16 @@ public boolean equals(Object obj) { return Objects.equals(name, other.getName()) && Objects.equals(assignmentMode, other.getAssignmentMode()) - && Objects.equals(keySerializer, other.getKeySerializer()) - && Objects.equals(valueSerializer, other.getValueSerializer()); + && Objects.equals(getKeySerializer(), other.getKeySerializer()) + && Objects.equals(getValueSerializer(), other.getValueSerializer()); } @Override public int hashCode() { int result = name.hashCode(); result = 31 * result + assignmentMode.hashCode(); - result = 31 * result + keySerializer.hashCode(); - result = 31 * result + valueSerializer.hashCode(); + result = 31 * result + getKeySerializer().hashCode(); + result = 31 * result + getValueSerializer().hashCode(); return result; } @@ -139,8 +177,8 @@ public int hashCode() { public String toString() { return "RegisteredBroadcastBackendStateMetaInfo{" + "name='" + name + '\'' + - ", keySerializer=" + keySerializer + - ", valueSerializer=" + valueSerializer + + ", keySerializer=" + getKeySerializer() + + ", valueSerializer=" + getValueSerializer() + ", assignmentMode=" + assignmentMode + '}'; } @@ -154,8 +192,12 @@ private StateMetaInfoSnapshot computeSnapshot() { Map> serializerConfigSnapshotsMap = new HashMap<>(2); String keySerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString(); String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(); + + TypeSerializer keySerializer = getKeySerializer(); serializerMap.put(keySerializerKey, keySerializer.duplicate()); serializerConfigSnapshotsMap.put(keySerializerKey, keySerializer.snapshotConfiguration()); + + TypeSerializer valueSerializer = getValueSerializer(); serializerMap.put(valueSerializerKey, valueSerializer.duplicate()); serializerConfigSnapshotsMap.put(valueSerializerKey, valueSerializer.snapshotConfiguration()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java index d05f31a0c5c76..b37c79de0264e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot; import org.apache.flink.util.Preconditions; @@ -44,18 +45,24 @@ public class RegisteredKeyValueStateBackendMetaInfo extends RegisteredStat @Nonnull private final StateDescriptor.Type stateType; @Nonnull - private final TypeSerializer namespaceSerializer; + private final StateSerializerProvider namespaceSerializerProvider; @Nonnull - private final TypeSerializer stateSerializer; + private final StateSerializerProvider stateSerializerProvider; @Nullable - private final StateSnapshotTransformer snapshotTransformer; + private StateSnapshotTransformer snapshotTransformer; public RegisteredKeyValueStateBackendMetaInfo( @Nonnull StateDescriptor.Type stateType, @Nonnull String name, @Nonnull TypeSerializer namespaceSerializer, @Nonnull TypeSerializer stateSerializer) { - this(stateType, name, namespaceSerializer, stateSerializer, null); + + this( + stateType, + name, + StateSerializerProvider.fromNewState(namespaceSerializer), + StateSerializerProvider.fromNewState(stateSerializer), + null); } public RegisteredKeyValueStateBackendMetaInfo( @@ -65,11 +72,12 @@ public RegisteredKeyValueStateBackendMetaInfo( @Nonnull TypeSerializer stateSerializer, @Nullable StateSnapshotTransformer snapshotTransformer) { - super(name); - this.stateType = stateType; - this.namespaceSerializer = namespaceSerializer; - this.stateSerializer = stateSerializer; - this.snapshotTransformer = snapshotTransformer; + this( + stateType, + name, + StateSerializerProvider.fromNewState(namespaceSerializer), + StateSerializerProvider.fromNewState(stateSerializer), + snapshotTransformer); } @SuppressWarnings("unchecked") @@ -77,13 +85,31 @@ public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sna this( StateDescriptor.Type.valueOf(snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE)), snapshot.getName(), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER)), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)), null); + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER))), + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))), + null); + Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.KEY_VALUE == snapshot.getBackendStateType()); } + private RegisteredKeyValueStateBackendMetaInfo( + @Nonnull StateDescriptor.Type stateType, + @Nonnull String name, + @Nonnull StateSerializerProvider namespaceSerializerProvider, + @Nonnull StateSerializerProvider stateSerializerProvider, + @Nullable StateSnapshotTransformer snapshotTransformer) { + + super(name); + this.stateType = stateType; + this.namespaceSerializerProvider = namespaceSerializerProvider; + this.stateSerializerProvider = stateSerializerProvider; + this.snapshotTransformer = snapshotTransformer; + } + @Nonnull public StateDescriptor.Type getStateType() { return stateType; @@ -91,12 +117,32 @@ public StateDescriptor.Type getStateType() { @Nonnull public TypeSerializer getNamespaceSerializer() { - return namespaceSerializer; + return namespaceSerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updateNamespaceSerializer(TypeSerializer newNamespaceSerializer) { + return namespaceSerializerProvider.registerNewSerializerForRestoredState(newNamespaceSerializer); + } + + @Nullable + public TypeSerializer getPreviousNamespaceSerializer() { + return namespaceSerializerProvider.previousSchemaSerializer(); } @Nonnull public TypeSerializer getStateSerializer() { - return stateSerializer; + return stateSerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updateStateSerializer(TypeSerializer newStateSerializer) { + return stateSerializerProvider.registerNewSerializerForRestoredState(newStateSerializer); + } + + @Nullable + public TypeSerializer getPreviousStateSerializer() { + return stateSerializerProvider.previousSchemaSerializer(); } @Nullable @@ -104,6 +150,10 @@ public StateSnapshotTransformer getSnapshotTransformer() { return snapshotTransformer; } + public void updateSnapshotTransformer(StateSnapshotTransformer snapshotTransformer) { + this.snapshotTransformer = snapshotTransformer; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -133,8 +183,8 @@ public String toString() { return "RegisteredKeyedBackendStateMetaInfo{" + "stateType=" + stateType + ", name='" + name + '\'' + - ", namespaceSerializer=" + namespaceSerializer + - ", stateSerializer=" + stateSerializer + + ", namespaceSerializer=" + getNamespaceSerializer() + + ", stateSerializer=" + getStateSerializer() + '}'; } @@ -153,34 +203,19 @@ public StateMetaInfoSnapshot snapshot() { return computeSnapshot(); } - public static void checkStateMetaInfo(StateMetaInfoSnapshot stateMetaInfoSnapshot, StateDescriptor stateDesc) { - Preconditions.checkState( - stateMetaInfoSnapshot != null, - "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," + - " but its corresponding restored snapshot cannot be found."); - - Preconditions.checkState(stateMetaInfoSnapshot.getBackendStateType() - == StateMetaInfoSnapshot.BackendStateType.KEY_VALUE, - "Incompatible state types. " + - "Was [" + stateMetaInfoSnapshot.getBackendStateType() + "], " + - "registered as [" + StateMetaInfoSnapshot.BackendStateType.KEY_VALUE + "]."); + public void checkStateMetaInfo(StateDescriptor stateDesc) { Preconditions.checkState( - Objects.equals(stateDesc.getName(), stateMetaInfoSnapshot.getName()), + Objects.equals(stateDesc.getName(), getName()), "Incompatible state names. " + - "Was [" + stateMetaInfoSnapshot.getName() + "], " + + "Was [" + getName() + "], " + "registered with [" + stateDesc.getName() + "]."); - final StateDescriptor.Type restoredType = - StateDescriptor.Type.valueOf( - stateMetaInfoSnapshot.getOption( - StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE)); - - if (stateDesc.getType() != StateDescriptor.Type.UNKNOWN && restoredType != StateDescriptor.Type.UNKNOWN) { + if (stateDesc.getType() != StateDescriptor.Type.UNKNOWN && getStateType() != StateDescriptor.Type.UNKNOWN) { Preconditions.checkState( - stateDesc.getType() == restoredType, + stateDesc.getType() == getStateType(), "Incompatible key/value state types. " + - "Was [" + restoredType + "], " + + "Was [" + getStateType() + "], " + "registered with [" + stateDesc.getType() + "]."); } } @@ -194,8 +229,12 @@ private StateMetaInfoSnapshot computeSnapshot() { Map> serializerConfigSnapshotsMap = new HashMap<>(2); String namespaceSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString(); String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(); + + TypeSerializer namespaceSerializer = getNamespaceSerializer(); serializerMap.put(namespaceSerializerKey, namespaceSerializer.duplicate()); serializerConfigSnapshotsMap.put(namespaceSerializerKey, namespaceSerializer.snapshotConfiguration()); + + TypeSerializer stateSerializer = getStateSerializer(); serializerMap.put(valueSerializerKey, stateSerializer.duplicate()); serializerConfigSnapshotsMap.put(valueSerializerKey, stateSerializer.snapshotConfiguration()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java index 10ba029605702..921947a4dd0bc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java @@ -19,11 +19,13 @@ package org.apache.flink.runtime.state; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot; import org.apache.flink.util.Preconditions; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Map; @@ -46,21 +48,22 @@ public class RegisteredOperatorStateBackendMetaInfo extends RegisteredStateMe * The type serializer for the elements in the state list */ @Nonnull - private final TypeSerializer partitionStateSerializer; + private final StateSerializerProvider partitionStateSerializerProvider; public RegisteredOperatorStateBackendMetaInfo( @Nonnull String name, @Nonnull TypeSerializer partitionStateSerializer, @Nonnull OperatorStateHandle.Mode assignmentMode) { - super(name); - this.partitionStateSerializer = partitionStateSerializer; - this.assignmentMode = assignmentMode; + this( + name, + StateSerializerProvider.fromNewState(partitionStateSerializer), + assignmentMode); } private RegisteredOperatorStateBackendMetaInfo(@Nonnull RegisteredOperatorStateBackendMetaInfo copy) { this( Preconditions.checkNotNull(copy).name, - copy.partitionStateSerializer.duplicate(), + copy.getPartitionStateSerializer().duplicate(), copy.assignmentMode); } @@ -68,13 +71,24 @@ private RegisteredOperatorStateBackendMetaInfo(@Nonnull RegisteredOperatorStateB public RegisteredOperatorStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) { this( snapshot.getName(), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)), + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))), OperatorStateHandle.Mode.valueOf( snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE))); + Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.OPERATOR == snapshot.getBackendStateType()); } + private RegisteredOperatorStateBackendMetaInfo( + @Nonnull String name, + @Nonnull StateSerializerProvider partitionStateSerializerProvider, + @Nonnull OperatorStateHandle.Mode assignmentMode) { + super(name); + this.partitionStateSerializerProvider = partitionStateSerializerProvider; + this.assignmentMode = assignmentMode; + } + /** * Creates a deep copy of the itself. */ @@ -96,7 +110,17 @@ public OperatorStateHandle.Mode getAssignmentMode() { @Nonnull public TypeSerializer getPartitionStateSerializer() { - return partitionStateSerializer; + return partitionStateSerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updatePartitionStateSerializer(TypeSerializer newPartitionStateSerializer) { + return partitionStateSerializerProvider.registerNewSerializerForRestoredState(newPartitionStateSerializer); + } + + @Nullable + public TypeSerializer getPreviousPartitionStateSerializer() { + return partitionStateSerializerProvider.previousSchemaSerializer(); } @Override @@ -112,7 +136,7 @@ public boolean equals(Object obj) { return (obj instanceof RegisteredOperatorStateBackendMetaInfo) && name.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getName()) && assignmentMode.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getAssignmentMode()) - && partitionStateSerializer.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getPartitionStateSerializer()); + && getPartitionStateSerializer().equals(((RegisteredOperatorStateBackendMetaInfo) obj).getPartitionStateSerializer()); } @Override @@ -128,7 +152,7 @@ public String toString() { return "RegisteredOperatorBackendStateMetaInfo{" + "name='" + name + "\'" + ", assignmentMode=" + assignmentMode + - ", partitionStateSerializer=" + partitionStateSerializer + + ", partitionStateSerializer=" + getPartitionStateSerializer() + '}'; } @@ -138,6 +162,8 @@ private StateMetaInfoSnapshot computeSnapshot() { StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE.toString(), assignmentMode.toString()); String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(); + + TypeSerializer partitionStateSerializer = getPartitionStateSerializer(); Map> serializerMap = Collections.singletonMap(valueSerializerKey, partitionStateSerializer.duplicate()); Map> serializerConfigSnapshotsMap = diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java index 0304b929c6d3f..961d96fa40577 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java @@ -19,11 +19,13 @@ package org.apache.flink.runtime.state; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot; import org.apache.flink.util.Preconditions; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Map; @@ -34,24 +36,34 @@ public class RegisteredPriorityQueueStateBackendMetaInfo extends RegisteredStateMetaInfoBase { @Nonnull - private final TypeSerializer elementSerializer; + private final StateSerializerProvider elementSerializerProvider; public RegisteredPriorityQueueStateBackendMetaInfo( @Nonnull String name, @Nonnull TypeSerializer elementSerializer) { - super(name); - this.elementSerializer = elementSerializer; + this(name, StateSerializerProvider.fromNewState(elementSerializer)); } @SuppressWarnings("unchecked") public RegisteredPriorityQueueStateBackendMetaInfo(StateMetaInfoSnapshot snapshot) { - this(snapshot.getName(), - (TypeSerializer) Preconditions.checkNotNull( - snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))); + this( + snapshot.getName(), + StateSerializerProvider.fromRestoredState( + (TypeSerializerSnapshot) Preconditions.checkNotNull( + snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)))); + Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE == snapshot.getBackendStateType()); } + private RegisteredPriorityQueueStateBackendMetaInfo( + @Nonnull String name, + @Nonnull StateSerializerProvider elementSerializerProvider) { + + super(name); + this.elementSerializerProvider = elementSerializerProvider; + } + @Nonnull @Override public StateMetaInfoSnapshot snapshot() { @@ -60,10 +72,21 @@ public StateMetaInfoSnapshot snapshot() { @Nonnull public TypeSerializer getElementSerializer() { - return elementSerializer; + return elementSerializerProvider.currentSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility updateElementSerializer(TypeSerializer newElementSerializer) { + return elementSerializerProvider.registerNewSerializerForRestoredState(newElementSerializer); + } + + @Nullable + public TypeSerializer getPreviousElementSerializer() { + return elementSerializerProvider.previousSchemaSerializer(); } private StateMetaInfoSnapshot computeSnapshot() { + TypeSerializer elementSerializer = getElementSerializer(); Map> serializerMap = Collections.singletonMap( StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(), @@ -82,6 +105,6 @@ private StateMetaInfoSnapshot computeSnapshot() { } public RegisteredPriorityQueueStateBackendMetaInfo deepCopy() { - return new RegisteredPriorityQueueStateBackendMetaInfo<>(name, elementSerializer.duplicate()); + return new RegisteredPriorityQueueStateBackendMetaInfo<>(name, getElementSerializer().duplicate()); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java index 4132d144a4adb..b7dff59aef036 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java @@ -42,4 +42,21 @@ public String getName() { @Nonnull public abstract StateMetaInfoSnapshot snapshot(); + + public static RegisteredStateMetaInfoBase fromMetaInfoSnapshot(@Nonnull StateMetaInfoSnapshot snapshot) { + + final StateMetaInfoSnapshot.BackendStateType backendStateType = snapshot.getBackendStateType(); + switch (backendStateType) { + case KEY_VALUE: + return new RegisteredKeyValueStateBackendMetaInfo<>(snapshot); + case OPERATOR: + return new RegisteredOperatorStateBackendMetaInfo<>(snapshot); + case BROADCAST: + return new RegisteredBroadcastStateBackendMetaInfo<>(snapshot); + case PRIORITY_QUEUE: + return new RegisteredPriorityQueueStateBackendMetaInfo<>(snapshot); + default: + throw new IllegalArgumentException("Unknown backend state type: " + backendStateType); + } + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java new file mode 100644 index 0000000000000..a24f12e42fbee --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A {@link StateSerializerProvider} wraps logic on how to obtain serializers for registered state, + * either with the previous schema of state in checkpoints or the current schema of state. + * + * @param the type of the state. + */ +@Internal +public abstract class StateSerializerProvider { + + /** + * The registered serializer for the state. + * + *

    In the case that this provider was created from a restored serializer snapshot via + * {@link #fromRestoredState(TypeSerializerSnapshot)}, but a new serializer was never registered + * for the state (i.e., this is the case if a restored state was never accessed), this would be {@code null}. + */ + @Nullable + TypeSerializer registeredSerializer; + + /** + * Creates a {@link StateSerializerProvider} for restored state from the previous serializer's snapshot. + * + *

    Once a new serializer is registered for the state, it should be provided via + * the {@link #registerNewSerializerForRestoredState(TypeSerializer)} method. + * + * @param stateSerializerSnapshot the previous serializer's snapshot. + * @param the type of the state. + * + * @return a new {@link StateSerializerProvider} for restored state. + */ + public static StateSerializerProvider fromRestoredState(TypeSerializerSnapshot stateSerializerSnapshot) { + return new RestoredStateSerializerProvider<>(stateSerializerSnapshot); + } + + /** + * Creates a {@link StateSerializerProvider} for new state from the registered state serializer. + * + * @param registeredStateSerializer the new state's registered serializer. + * @param the type of the state. + * + * @return a new {@link StateSerializerProvider} for new state. + */ + public static StateSerializerProvider fromNewState(TypeSerializer registeredStateSerializer) { + return new NewStateSerializerProvider<>(registeredStateSerializer); + } + + private StateSerializerProvider(@Nullable TypeSerializer stateSerializer) { + this.registeredSerializer = stateSerializer; + } + + /** + * Gets the serializer that recognizes the current serialization schema of the state. + * This is the serializer that should be used for regular state serialization and + * deserialization after state has been restored. + * + *

    If this provider was created from a restored state's serializer snapshot, while a + * new serializer (with a new schema) was not registered for the state (i.e., because + * the state was never accessed after it was restored), then the schema of state remains + * identical. Therefore, in this case, it is guaranteed that the serializer returned by + * this method is the same as the one returned by {@link #previousSchemaSerializer()}. + * + *

    If this provider was created from new state, then this always returns the + * serializer that the new state was registered with. + * + * @return a serializer that reads and writes in the current schema of the state. + */ + @Nonnull + public abstract TypeSerializer currentSchemaSerializer(); + + /** + * Gets the serializer that recognizes the previous serialization schema of the state. + * This is the serializer that should be used for restoring the state, i.e. when the state + * is still in the previous serialization schema. + * + *

    This method can only be used if this provider was created from a restored state's serializer + * snapshot. If this provider was created from new state, then this method is + * irrelevant, since there doesn't exist any previous version of the state schema. + * + * @return a serializer that reads and writes in the previous schema of the state. + */ + @Nonnull + public abstract TypeSerializer previousSchemaSerializer(); + + /** + * For restored state, register a new serializer that potentially has a new serialization schema. + * + *

    Users are allowed to register serializers for state only once. Therefore, this method + * is irrelevant if this provider was created from new state, since a state serializer had + * been registered already. + * + *

    For the case where this provider was created from restored state, then this method should + * be called at most once. The new serializer will be checked for its schema compatibility with the + * previous serializer's schema, and returned to the caller. The caller is responsible for + * checking the result and react appropriately to it, as follows: + *

    + * + * @return the schema compatibility of the new registered serializer, with respect to the previous serializer. + */ + @Nonnull + public abstract TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer); + + /** + * Implementation of the {@link StateSerializerProvider} for the restored state case. + */ + private static class RestoredStateSerializerProvider extends StateSerializerProvider { + + /** + * The snapshot of the previous serializer of the state. + */ + @Nonnull + private final TypeSerializerSnapshot previousSerializerSnapshot; + + private boolean isRegisteredWithIncompatibleSerializer = false; + + RestoredStateSerializerProvider(TypeSerializerSnapshot previousSerializerSnapshot) { + super(null); + this.previousSerializerSnapshot = Preconditions.checkNotNull(previousSerializerSnapshot); + } + + /** + * The restore serializer, lazily created only when the restore serializer is accessed. + * + *

    NOTE: It is important to only create this lazily, so that off-heap + * state do not fail eagerly when restoring state that has a + * {@link UnloadableDummyTypeSerializer} as the previous serializer. This should + * be relevant only for restores from Flink versions prior to 1.7.x. + */ + @Nullable + private TypeSerializer cachedRestoredSerializer; + + @Override + @Nonnull + public TypeSerializer currentSchemaSerializer() { + if (registeredSerializer != null) { + checkState( + !isRegisteredWithIncompatibleSerializer, + "Unable to provide a serializer with the current schema, because the restored state was " + + "registered with a new serializer that has incompatible schema."); + + return registeredSerializer; + } + + // if we are not yet registered with a new serializer, + // we can just use the restore serializer to read / write the state. + return previousSchemaSerializer(); + } + + @Nonnull + public TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer) { + checkNotNull(newSerializer); + if (registeredSerializer != null) { + throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); + } + + TypeSerializerSchemaCompatibility result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); + if (result.isIncompatible()) { + this.isRegisteredWithIncompatibleSerializer = true; + } + this.registeredSerializer = newSerializer; + return result; + } + + @Nonnull + public final TypeSerializer previousSchemaSerializer() { + if (cachedRestoredSerializer != null) { + return cachedRestoredSerializer; + } + + this.cachedRestoredSerializer = previousSerializerSnapshot.restoreSerializer(); + return cachedRestoredSerializer; + } + } + + /** + * Implementation of the {@link StateSerializerProvider} for the new state case. + */ + private static class NewStateSerializerProvider extends StateSerializerProvider { + + NewStateSerializerProvider(TypeSerializer registeredStateSerializer) { + super(Preconditions.checkNotNull(registeredStateSerializer)); + } + + @Override + @Nonnull + @SuppressWarnings("ConstantConditions") + public TypeSerializer currentSchemaSerializer() { + return registeredSerializer; + } + + @Override + @Nonnull + public TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer) { + throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); + } + + @Override + @Nonnull + public TypeSerializer previousSchemaSerializer() { + throw new UnsupportedOperationException("This is a NewStateSerializerProvider; you cannot get a restore serializer because there was no restored state."); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 4eff3a285bbdc..3f8761b657a3f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -30,7 +30,6 @@ import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.fs.FSDataInputStream; import org.apache.flink.core.memory.DataInputViewStreamWrapper; @@ -79,6 +78,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; @@ -125,14 +125,6 @@ public class HeapKeyedStateBackend extends AbstractKeyedStateBackend { */ private final Map registeredPQStates; - /** - * Map of state names to their corresponding restored state meta info. - * - *

    TODO this map can be removed when eager-state registration is in place. - * TODO we currently need this cached to check state migration strategies when new serializers are registered. - */ - private final Map restoredStateMetaInfo; - /** * The configuration for local recovery. */ @@ -173,7 +165,6 @@ public HeapKeyedStateBackend( this.snapshotStrategy = new HeapSnapshotStrategy(synchronicityTrait); LOG.info("Initializing heap keyed state backend with stream factory."); - this.restoredStateMetaInfo = new HashMap<>(); this.priorityQueueSetFactory = priorityQueueSetFactory; } @@ -194,23 +185,9 @@ public KeyGrou // TODO we implement the simple way of supporting the current functionality, mimicking keyed state // because this should be reworked in FLINK-9376 and then we should have a common algorithm over // StateMetaInfoSnapshot that avoids this code duplication. - StateMetaInfoSnapshot restoredMetaInfoSnapshot = - restoredStateMetaInfo.get(StateUID.of(stateName, StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE)); - - Preconditions.checkState( - restoredMetaInfoSnapshot != null, - "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," + - " but its corresponding restored snapshot cannot be found."); - - StateMetaInfoSnapshot.CommonSerializerKeys serializerKey = - StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER; - - @SuppressWarnings("unchecked") - TypeSerializerSnapshot serializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(serializerKey)); TypeSerializerSchemaCompatibility compatibilityResult = - serializerSnapshot.resolveSchemaCompatibility(byteOrderedElementSerializer); + existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer); if (compatibilityResult.isIncompatible()) { throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible.")); @@ -252,57 +229,42 @@ private KeyGro private StateTable tryRegisterStateTable( TypeSerializer namespaceSerializer, StateDescriptor stateDesc, - StateSnapshotTransformer snapshotTransformer) throws StateMigrationException { + @Nullable StateSnapshotTransformer snapshotTransformer) throws StateMigrationException { @SuppressWarnings("unchecked") StateTable stateTable = (StateTable) registeredKVStates.get(stateDesc.getName()); TypeSerializer newStateSerializer = stateDesc.getSerializer(); - RegisteredKeyValueStateBackendMetaInfo newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( - stateDesc.getType(), - stateDesc.getName(), - namespaceSerializer, - newStateSerializer, - snapshotTransformer); if (stateTable != null) { - @SuppressWarnings("unchecked") - StateMetaInfoSnapshot restoredMetaInfoSnapshot = - restoredStateMetaInfo.get( - StateUID.of(stateDesc.getName(), StateMetaInfoSnapshot.BackendStateType.KEY_VALUE)); - - Preconditions.checkState( - restoredMetaInfoSnapshot != null, - "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," + - " but its corresponding restored snapshot cannot be found."); + RegisteredKeyValueStateBackendMetaInfo restoredKvMetaInfo = stateTable.getMetaInfo(); - @SuppressWarnings("unchecked") - TypeSerializerSnapshot namespaceSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot( - StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString())); + restoredKvMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility namespaceCompatibility = - namespaceSerializerSnapshot.resolveSchemaCompatibility(namespaceSerializer); - if (namespaceCompatibility.isIncompatible()) { - throw new StateMigrationException("For heap backends, the new namespace serializer must not be incompatible."); + restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer); + if (!namespaceCompatibility.isCompatibleAsIs()) { + throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible."); } - @SuppressWarnings("unchecked") - TypeSerializerSnapshot stateSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot( - StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString())); - - RegisteredKeyValueStateBackendMetaInfo.checkStateMetaInfo(restoredMetaInfoSnapshot, stateDesc); + restoredKvMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility stateCompatibility = - stateSerializerSnapshot.resolveSchemaCompatibility(newStateSerializer); + restoredKvMetaInfo.updateStateSerializer(newStateSerializer); if (stateCompatibility.isIncompatible()) { throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible."); } - stateTable.setMetaInfo(newMetaInfo); + stateTable.setMetaInfo(restoredKvMetaInfo); } else { + RegisteredKeyValueStateBackendMetaInfo newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( + stateDesc.getType(), + stateDesc.getName(), + namespaceSerializer, + newStateSerializer, + snapshotTransformer); + stateTable = snapshotStrategy.newStateTable(newMetaInfo); registeredKVStates.put(stateDesc.getName(), stateTable); } @@ -536,10 +498,6 @@ private void createOrCheckStateForMetaInfo( Map kvStatesById) { for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) { - restoredStateMetaInfo.put( - StateUID.of(metaInfoSnapshot.getName(), metaInfoSnapshot.getBackendStateType()), - metaInfoSnapshot); - final StateSnapshotRestore registeredState; switch (metaInfoSnapshot.getBackendStateType()) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java index 77c267adff1d0..836edef0aac20 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java @@ -132,11 +132,6 @@ static class OperatorBackendStateMetaInfoReaderV2V3 implements StateMetaInfoRead static final OperatorBackendStateMetaInfoReaderV2V3 INSTANCE = new OperatorBackendStateMetaInfoReaderV2V3(); - private static final String[] ORDERED_KEY_STRINGS = - new String[]{ - StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString(), - StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString()}; - @Nonnull @Override public StateMetaInfoSnapshot readStateMetaInfoSnapshot( @@ -156,17 +151,25 @@ public StateMetaInfoSnapshot readStateMetaInfoSnapshot( final int listSize = stateSerializerAndConfigList.size(); StateMetaInfoSnapshot.BackendStateType stateType = listSize == 1 ? StateMetaInfoSnapshot.BackendStateType.OPERATOR : StateMetaInfoSnapshot.BackendStateType.BROADCAST; - Map> serializerConfigsMap = new HashMap<>(listSize); - for (int i = 0; i < listSize; ++i) { - Tuple2, TypeSerializerSnapshot> serializerAndConf = - stateSerializerAndConfigList.get(i); - - // this particular mapping happens to support both, V2 and V3 - String serializerKey = ORDERED_KEY_STRINGS[ORDERED_KEY_STRINGS.length - 1 - i]; - serializerConfigsMap.put( - serializerKey, - serializerAndConf.f1); + Map> serializerConfigsMap = new HashMap<>(listSize); + switch (stateType) { + case OPERATOR: + serializerConfigsMap.put( + StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(), + stateSerializerAndConfigList.get(0).f1); + break; + case BROADCAST: + serializerConfigsMap.put( + StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString(), + stateSerializerAndConfigList.get(0).f1); + + serializerConfigsMap.put( + StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(), + stateSerializerAndConfigList.get(1).f1); + break; + default: + throw new IllegalStateException("Unknown operator state type " + stateType); } return new StateMetaInfoSnapshot( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java index 1e9d9191079c5..9b05500e4d017 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java @@ -81,7 +81,7 @@ public enum CommonSerializerKeys { /** The configurations of all the type serializers used with the state. */ @Nonnull - private final Map> serializerConfigSnapshots; + private final Map> serializerSnapshots; // TODO this will go away once all serializers have the restoreSerializer() factory method properly implemented. /** The serializers used by the state. */ @@ -92,8 +92,8 @@ public StateMetaInfoSnapshot( @Nonnull String name, @Nonnull BackendStateType backendStateType, @Nonnull Map options, - @Nonnull Map> serializerConfigSnapshots) { - this(name, backendStateType, options, serializerConfigSnapshots, new HashMap<>()); + @Nonnull Map> serializerSnapshots) { + this(name, backendStateType, options, serializerSnapshots, new HashMap<>()); } /** @@ -106,12 +106,12 @@ public StateMetaInfoSnapshot( @Nonnull String name, @Nonnull BackendStateType backendStateType, @Nonnull Map options, - @Nonnull Map> serializerConfigSnapshots, + @Nonnull Map> serializerSnapshots, @Nonnull Map> serializers) { this.name = name; this.backendStateType = backendStateType; this.options = options; - this.serializerConfigSnapshots = serializerConfigSnapshots; + this.serializerSnapshots = serializerSnapshots; this.serializers = serializers; } @@ -121,13 +121,13 @@ public BackendStateType getBackendStateType() { } @Nullable - public TypeSerializerSnapshot getTypeSerializerConfigSnapshot(@Nonnull String key) { - return serializerConfigSnapshots.get(key); + public TypeSerializerSnapshot getTypeSerializerSnapshot(@Nonnull String key) { + return serializerSnapshots.get(key); } @Nullable - public TypeSerializerSnapshot getTypeSerializerConfigSnapshot(@Nonnull CommonSerializerKeys key) { - return getTypeSerializerConfigSnapshot(key.toString()); + public TypeSerializerSnapshot getTypeSerializerSnapshot(@Nonnull CommonSerializerKeys key) { + return getTypeSerializerSnapshot(key.toString()); } @Nullable @@ -150,20 +150,9 @@ public String getName() { return name; } - @Nullable - public TypeSerializer restoreTypeSerializer(@Nonnull String key) { - TypeSerializerSnapshot configSnapshot = getTypeSerializerConfigSnapshot(key); - return (configSnapshot != null) ? configSnapshot.restoreSerializer() : null; - } - - @Nullable - public TypeSerializer restoreTypeSerializer(@Nonnull CommonSerializerKeys key) { - return restoreTypeSerializer(key.toString()); - } - @Nonnull - public Map> getSerializerConfigSnapshotsImmutable() { - return Collections.unmodifiableMap(serializerConfigSnapshots); + public Map> getSerializerSnapshotsImmutable() { + return Collections.unmodifiableMap(serializerSnapshots); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java index 4408dfcacefdb..ad1e7be287108 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java @@ -165,7 +165,7 @@ public void writeStateMetaInfoSnapshot( @Nonnull DataOutputView outputView) throws IOException { final Map optionsMap = snapshot.getOptionsImmutable(); final Map> serializerConfigSnapshotsMap = - snapshot.getSerializerConfigSnapshotsImmutable(); + snapshot.getSerializerSnapshotsImmutable(); outputView.writeUTF(snapshot.getName()); outputView.writeInt(snapshot.getBackendStateType().ordinal()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlStateFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlStateFactory.java index 45f4e3bf143dd..0a881c709781c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlStateFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlStateFactory.java @@ -188,15 +188,17 @@ private StateSnapshotTransformFactory getSnapshotTransformFactory() { } } - /** Serializer for user state value with TTL. */ - private static class TtlSerializer extends CompositeSerializer> { + /** + * Serializer for user state value with TTL. Visibility is public for usage with external tools. + */ + public static class TtlSerializer extends CompositeSerializer> { private static final long serialVersionUID = 131020282727167064L; - TtlSerializer(TypeSerializer userValueSerializer) { + public TtlSerializer(TypeSerializer userValueSerializer) { super(true, LongSerializer.INSTANCE, userValueSerializer); } - TtlSerializer(PrecomputedParameters precomputed, TypeSerializer ... fieldSerializers) { + public TtlSerializer(PrecomputedParameters precomputed, TypeSerializer ... fieldSerializers) { super(precomputed, fieldSerializers); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlValue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlValue.java index 48435d567e86e..dba06d3a88b0c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlValue.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ttl/TtlValue.java @@ -23,28 +23,28 @@ import java.io.Serializable; /** - * This class wraps user value of state with TTL. + * This class wraps user value of state with TTL. Visibility is public for usage with external tools. * * @param Type of the user value of state with TTL */ -class TtlValue implements Serializable { +public class TtlValue implements Serializable { private static final long serialVersionUID = 5221129704201125020L; @Nullable private final T userValue; private final long lastAccessTimestamp; - TtlValue(@Nullable T userValue, long lastAccessTimestamp) { + public TtlValue(@Nullable T userValue, long lastAccessTimestamp) { this.userValue = userValue; this.lastAccessTimestamp = lastAccessTimestamp; } @Nullable - T getUserValue() { + public T getUserValue() { return userValue; } - long getLastAccessTimestamp() { + public long getLastAccessTimestamp() { return lastAccessTimestamp; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java index 94ecaffa4a820..9d8437a682841 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java @@ -34,7 +34,7 @@ public class SlotReport implements Serializable, Iterable { private static final long serialVersionUID = -3150175198722481689L; - /** The slots status of the TaskManager */ + /** The slots status of the TaskManager. */ private final Collection slotsStatus; public SlotReport() { @@ -53,4 +53,11 @@ public SlotReport(final Collection slotsStatus) { public Iterator iterator() { return slotsStatus.iterator(); } + + @Override + public String toString() { + return "SlotReport{" + + "slotsStatus=" + slotsStatus + + '}'; + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java index a3bc4a6af4a9e..83345c35c69a5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java @@ -34,16 +34,16 @@ public class SlotStatus implements Serializable { private static final long serialVersionUID = 5099191707339664493L; - /** slotID to identify a slot */ + /** SlotID to identify a slot. */ private final SlotID slotID; - /** the resource profile of the slot */ + /** The resource profile of the slot. */ private final ResourceProfile resourceProfile; - /** if the slot is allocated, allocationId identify its allocation; else, allocationId is null */ + /** If the slot is allocated, allocationId identify its allocation; else, allocationId is null. */ private final AllocationID allocationID; - /** if the slot is allocated, jobId identify which job this slot is allocated to; else, jobId is null */ + /** If the slot is allocated, jobId identify which job this slot is allocated to; else, jobId is null. */ private final JobID jobID; public SlotStatus(SlotID slotID, ResourceProfile resourceProfile) { @@ -51,10 +51,10 @@ public SlotStatus(SlotID slotID, ResourceProfile resourceProfile) { } public SlotStatus( - SlotID slotID, - ResourceProfile resourceProfile, - JobID jobID, - AllocationID allocationID) { + SlotID slotID, + ResourceProfile resourceProfile, + JobID jobID, + AllocationID allocationID) { this.slotID = checkNotNull(slotID, "slotID cannot be null"); this.resourceProfile = checkNotNull(resourceProfile, "profile cannot be null"); this.allocationID = allocationID; @@ -62,7 +62,7 @@ public SlotStatus( } /** - * Get the unique identification of this slot + * Get the unique identification of this slot. * * @return The slot id */ @@ -71,7 +71,7 @@ public SlotID getSlotID() { } /** - * Get the resource profile of this slot + * Get the resource profile of this slot. * * @return The resource profile */ @@ -80,7 +80,7 @@ public ResourceProfile getResourceProfile() { } /** - * Get the allocation id of this slot + * Get the allocation id of this slot. * * @return The allocation id if this slot is allocated, otherwise null */ @@ -89,7 +89,7 @@ public AllocationID getAllocationID() { } /** - * Get the job id of the slot allocated for + * Get the job id of the slot allocated for. * * @return The job id if this slot is allocated, otherwise null */ @@ -130,4 +130,13 @@ public int hashCode() { return result; } + @Override + public String toString() { + return "SlotStatus{" + + "slotID=" + slotID + + ", resourceProfile=" + resourceProfile + + ", allocationID=" + allocationID + + ", jobID=" + jobID + + '}'; + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java index aba8bda191825..10c9e2f123ac1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointMetrics; import org.apache.flink.runtime.checkpoint.TaskStateSnapshot; import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.taskmanager.CheckpointResponder; import org.apache.flink.util.Preconditions; @@ -54,9 +55,12 @@ public void acknowledgeCheckpoint( public void declineCheckpoint( JobID jobID, ExecutionAttemptID executionAttemptID, - long checkpointId, + long checkpointId, Throwable cause) { - checkpointCoordinatorGateway.declineCheckpoint(jobID, executionAttemptID, checkpointId, cause); + checkpointCoordinatorGateway.declineCheckpoint(new DeclineCheckpoint(jobID, + executionAttemptID, + checkpointId, + cause)); } } diff --git a/flink-runtime/src/main/resources/META-INF/NOTICE b/flink-runtime/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..12cd1d4ccf5b4 --- /dev/null +++ b/flink-runtime/src/main/resources/META-INF/NOTICE @@ -0,0 +1,47 @@ +flink-runtime +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.typesafe.akka:akka-remote_2.11:2.4.20 +- io.netty:netty:3.10.6.Final +- org.apache.zookeeper:zookeeper:3.4.10 +- org.uncommons.maths:uncommons-maths:1.2.2a + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit diff --git a/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 0000000000000..31ebc840539c1 --- /dev/null +++ b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 0000000000000..b1c292b54cb2b --- /dev/null +++ b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 0000000000000..29ad562af0a5e --- /dev/null +++ b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 0000000000000..ec5f348998bd4 --- /dev/null +++ b/flink-runtime/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java index cf1e7f7f82d04..2d09b46464ff7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java @@ -21,15 +21,16 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; +/** Test checkpoint statistics counters. */ public class CheckpointStatsCountsTest { /** * Tests that counts are reported correctly. */ @Test - public void testCounts() throws Exception { + public void testCounts() { CheckpointStatsCounts counts = new CheckpointStatsCounts(); assertEquals(0, counts.getNumberOfRestoredCheckpoints()); assertEquals(0, counts.getTotalNumberOfCheckpoints()); @@ -80,19 +81,15 @@ public void testCounts() throws Exception { * incrementing the in progress checkpoints before throws an Exception. */ @Test - public void testCompleteOrFailWithoutInProgressCheckpoint() throws Exception { + public void testCompleteOrFailWithoutInProgressCheckpoint() { CheckpointStatsCounts counts = new CheckpointStatsCounts(); - try { - counts.incrementCompletedCheckpoints(); - fail("Did not throw expected Exception"); - } catch (IllegalStateException ignored) { - } - - try { - counts.incrementFailedCheckpoints(); - fail("Did not throw expected Exception"); - } catch (IllegalStateException ignored) { - } + counts.incrementCompletedCheckpoints(); + assertTrue("Number of checkpoints in progress should never be negative", + counts.getNumberOfInProgressCheckpoints() >= 0); + + counts.incrementFailedCheckpoints(); + assertTrue("Number of checkpoints in progress should never be negative", + counts.getNumberOfInProgressCheckpoints() >= 0); } /** @@ -100,7 +97,7 @@ public void testCompleteOrFailWithoutInProgressCheckpoint() throws Exception { * parent. */ @Test - public void testCreateSnapshot() throws Exception { + public void testCreateSnapshot() { CheckpointStatsCounts counts = new CheckpointStatsCounts(); counts.incrementRestoredCheckpoints(); counts.incrementRestoredCheckpoints(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/classloading/ClassLoaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/classloading/ClassLoaderTest.java index c02278c508887..7c664ce0b3206 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/classloading/ClassLoaderTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/classloading/ClassLoaderTest.java @@ -19,21 +19,78 @@ package org.apache.flink.runtime.classloading; import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; +import org.apache.flink.runtime.rpc.messages.RemoteRpcInvocation; +import org.apache.flink.testutils.ClassLoaderUtils; +import org.apache.flink.util.SerializedValue; import org.apache.flink.util.TestLogger; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import java.net.URL; import java.net.URLClassLoader; +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.isA; +import static org.hamcrest.Matchers.hasItemInArray; +import static org.hamcrest.Matchers.hasProperty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; /** - * Tests for classloading and class loder utilities. + * Tests for classloading and class loader utilities. */ public class ClassLoaderTest extends TestLogger { + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testMessageDecodingWithUnavailableClass() throws Exception { + final ClassLoader systemClassLoader = getClass().getClassLoader(); + + final String className = "UserClass"; + final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava( + temporaryFolder.newFolder(), + className + ".java", + "import java.io.Serializable;\n" + + "public class " + className + " implements Serializable {}"); + + RemoteRpcInvocation method = new RemoteRpcInvocation( + "test", + new Class[] { + int.class, + Class.forName(className, false, userClassLoader)}, + new Object[] { + 1, + Class.forName(className, false, userClassLoader).newInstance()}); + + SerializedValue serializedMethod = new SerializedValue<>(method); + + expectedException.expect(ClassNotFoundException.class); + expectedException.expect( + allOf( + isA(ClassNotFoundException.class), + hasProperty("suppressed", + hasItemInArray( + allOf( + isA(ClassNotFoundException.class), + hasProperty("message", + containsString("Could not deserialize 1th parameter type of method test(int, ...)."))))))); + + RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader); + deserializedMethod.getMethodName(); + + userClassLoader.close(); + } + @Test public void testParentFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/ConjunctFutureTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/ConjunctFutureTest.java index 9f33866bf1049..28fa8fb4c1e14 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/ConjunctFutureTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/ConjunctFutureTest.java @@ -18,21 +18,27 @@ package org.apache.flink.runtime.concurrent; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture; import org.apache.flink.util.TestLogger; -import org.hamcrest.collection.IsIterableContainingInAnyOrder; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -40,7 +46,7 @@ import static org.junit.Assert.fail; /** - * Tests for the {@link ConjunctFuture} and {@link FutureUtils.WaitingConjunctFuture}. + * Tests for the {@link ConjunctFuture} and its sub classes. */ @RunWith(Parameterized.class) public class ConjunctFutureTest extends TestLogger { @@ -193,23 +199,33 @@ public void testConjunctFutureFailureOnSuccessive() throws Exception { } /** - * Tests that the conjunct future returns upon completion the collection of all future values. + * Tests that the conjunct future returns upon completion the collection of all future values + * in the same order in which the futures were inserted. */ @Test - public void testConjunctFutureValue() throws ExecutionException, InterruptedException { - java.util.concurrent.CompletableFuture future1 = java.util.concurrent.CompletableFuture.completedFuture(1); - java.util.concurrent.CompletableFuture future2 = java.util.concurrent.CompletableFuture.completedFuture(2L); - java.util.concurrent.CompletableFuture future3 = new java.util.concurrent.CompletableFuture<>(); + public void testConjunctFutureValue() throws Exception { + final int numberFutures = 10; - ConjunctFuture> result = FutureUtils.combineAll(Arrays.asList(future1, future2, future3)); + final List> futures = new ArrayList<>(numberFutures); + for (int i = 0; i < numberFutures; i++) { + futures.add(new CompletableFuture<>()); + } - assertFalse(result.isDone()); + ConjunctFuture> result = FutureUtils.combineAll(futures); - future3.complete(.1); + final List>> shuffledFutures = IntStream.range(0, futures.size()) + .mapToObj(index -> Tuple2.of(index, futures.get(index))) + .collect(Collectors.toList()); + Collections.shuffle(shuffledFutures); - assertTrue(result.isDone()); + for (Tuple2> shuffledFuture : shuffledFutures) { + assertThat(result.isDone(), is(false)); + shuffledFuture.f1.complete(shuffledFuture.f0); + } + + assertThat(result.isDone(), is(true)); - assertThat(result.get(), IsIterableContainingInAnyOrder.containsInAnyOrder(1, 2L, .1)); + assertThat(result.get(), is(equalTo(IntStream.range(0, numberFutures).boxed().collect(Collectors.toList())))); } @Test diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java index e20d34b1d3e1f..a617ce1e5152d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java @@ -33,87 +33,115 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.operators.BatchTask; import org.apache.flink.util.SerializedValue; +import org.apache.flink.util.TestLogger; import org.junit.Test; +import javax.annotation.Nonnull; + +import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** * Tests for the {@link TaskDeploymentDescriptor}. */ -public class TaskDeploymentDescriptorTest { +public class TaskDeploymentDescriptorTest extends TestLogger { + + private static final JobID jobID = new JobID(); + private static final JobVertexID vertexID = new JobVertexID(); + private static final ExecutionAttemptID execId = new ExecutionAttemptID(); + private static final AllocationID allocationId = new AllocationID(); + private static final String jobName = "job name"; + private static final String taskName = "task name"; + private static final int numberOfKeyGroups = 1; + private static final int indexInSubtaskGroup = 0; + private static final int currentNumberOfSubtasks = 1; + private static final int attemptNumber = 0; + private static final Configuration jobConfiguration = new Configuration(); + private static final Configuration taskConfiguration = new Configuration(); + private static final Class invokableClass = BatchTask.class; + private static final List producedResults = new ArrayList(0); + private static final List inputGates = new ArrayList(0); + private static final List requiredJars = new ArrayList<>(0); + private static final List requiredClasspaths = new ArrayList<>(0); + private static final int targetSlotNumber = 47; + private static final TaskStateSnapshot taskStateHandles = new TaskStateSnapshot(); + private static final JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(1L, taskStateHandles); + + private final SerializedValue executionConfig = new SerializedValue<>(new ExecutionConfig()); + private final SerializedValue serializedJobInformation = new SerializedValue<>(new JobInformation( + jobID, jobName, executionConfig, jobConfiguration, requiredJars, requiredClasspaths)); + private final SerializedValue serializedJobVertexInformation = new SerializedValue<>(new TaskInformation( + vertexID, taskName, currentNumberOfSubtasks, numberOfKeyGroups, invokableClass.getName(), taskConfiguration)); + + public TaskDeploymentDescriptorTest() throws IOException {} + + @Test + public void testSerialization() throws Exception { + final TaskDeploymentDescriptor orig = createTaskDeploymentDescriptor( + new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation), + new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation)); + + final TaskDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig); + + assertFalse(orig.getSerializedJobInformation() == copy.getSerializedJobInformation()); + assertFalse(orig.getSerializedTaskInformation() == copy.getSerializedTaskInformation()); + assertFalse(orig.getExecutionAttemptId() == copy.getExecutionAttemptId()); + assertFalse(orig.getTaskRestore() == copy.getTaskRestore()); + assertFalse(orig.getProducedPartitions() == copy.getProducedPartitions()); + assertFalse(orig.getInputGates() == copy.getInputGates()); + + assertEquals(orig.getSerializedJobInformation(), copy.getSerializedJobInformation()); + assertEquals(orig.getSerializedTaskInformation(), copy.getSerializedTaskInformation()); + assertEquals(orig.getExecutionAttemptId(), copy.getExecutionAttemptId()); + assertEquals(orig.getAllocationId(), copy.getAllocationId()); + assertEquals(orig.getSubtaskIndex(), copy.getSubtaskIndex()); + assertEquals(orig.getAttemptNumber(), copy.getAttemptNumber()); + assertEquals(orig.getTargetSlotNumber(), copy.getTargetSlotNumber()); + assertEquals(orig.getTaskRestore().getRestoreCheckpointId(), copy.getTaskRestore().getRestoreCheckpointId()); + assertEquals(orig.getTaskRestore().getTaskStateSnapshot(), copy.getTaskRestore().getTaskStateSnapshot()); + assertEquals(orig.getProducedPartitions(), copy.getProducedPartitions()); + assertEquals(orig.getInputGates(), copy.getInputGates()); + } + @Test - public void testSerialization() { + public void testOffLoadedAndNonOffLoadedPayload() { + final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor( + new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation), + new TaskDeploymentDescriptor.Offloaded<>(new PermanentBlobKey())); + + SerializedValue actualSerializedJobInformation = taskDeploymentDescriptor.getSerializedJobInformation(); + assertThat(actualSerializedJobInformation, is(serializedJobInformation)); + try { - final JobID jobID = new JobID(); - final JobVertexID vertexID = new JobVertexID(); - final ExecutionAttemptID execId = new ExecutionAttemptID(); - final AllocationID allocationId = new AllocationID(); - final String jobName = "job name"; - final String taskName = "task name"; - final int numberOfKeyGroups = 1; - final int indexInSubtaskGroup = 0; - final int currentNumberOfSubtasks = 1; - final int attemptNumber = 0; - final Configuration jobConfiguration = new Configuration(); - final Configuration taskConfiguration = new Configuration(); - final Class invokableClass = BatchTask.class; - final List producedResults = new ArrayList(0); - final List inputGates = new ArrayList(0); - final List requiredJars = new ArrayList<>(0); - final List requiredClasspaths = new ArrayList<>(0); - final SerializedValue executionConfig = new SerializedValue<>(new ExecutionConfig()); - final SerializedValue serializedJobInformation = new SerializedValue<>(new JobInformation( - jobID, jobName, executionConfig, jobConfiguration, requiredJars, requiredClasspaths)); - final SerializedValue serializedJobVertexInformation = new SerializedValue<>(new TaskInformation( - vertexID, taskName, currentNumberOfSubtasks, numberOfKeyGroups, invokableClass.getName(), taskConfiguration)); - final int targetSlotNumber = 47; - final TaskStateSnapshot taskStateHandles = new TaskStateSnapshot(); - final JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(1L, taskStateHandles); - - final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor( - jobID, - new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation), - new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation), - execId, - allocationId, - indexInSubtaskGroup, - attemptNumber, - targetSlotNumber, - taskRestore, - producedResults, - inputGates); - - final TaskDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig); - - assertFalse(orig.getSerializedJobInformation() == copy.getSerializedJobInformation()); - assertFalse(orig.getSerializedTaskInformation() == copy.getSerializedTaskInformation()); - assertFalse(orig.getExecutionAttemptId() == copy.getExecutionAttemptId()); - assertFalse(orig.getTaskRestore() == copy.getTaskRestore()); - assertFalse(orig.getProducedPartitions() == copy.getProducedPartitions()); - assertFalse(orig.getInputGates() == copy.getInputGates()); - - assertEquals(orig.getSerializedJobInformation(), copy.getSerializedJobInformation()); - assertEquals(orig.getSerializedTaskInformation(), copy.getSerializedTaskInformation()); - assertEquals(orig.getExecutionAttemptId(), copy.getExecutionAttemptId()); - assertEquals(orig.getAllocationId(), copy.getAllocationId()); - assertEquals(orig.getSubtaskIndex(), copy.getSubtaskIndex()); - assertEquals(orig.getAttemptNumber(), copy.getAttemptNumber()); - assertEquals(orig.getTargetSlotNumber(), copy.getTargetSlotNumber()); - assertEquals(orig.getTaskRestore().getRestoreCheckpointId(), copy.getTaskRestore().getRestoreCheckpointId()); - assertEquals(orig.getTaskRestore().getTaskStateSnapshot(), copy.getTaskRestore().getTaskStateSnapshot()); - assertEquals(orig.getProducedPartitions(), copy.getProducedPartitions()); - assertEquals(orig.getInputGates(), copy.getInputGates()); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); + taskDeploymentDescriptor.getSerializedTaskInformation(); + fail("Expected to fail since the task information should be offloaded."); + } catch (IllegalStateException expected) { + // expected } } + + @Nonnull + private TaskDeploymentDescriptor createTaskDeploymentDescriptor(TaskDeploymentDescriptor.MaybeOffloaded jobInformation, TaskDeploymentDescriptor.MaybeOffloaded taskInformation) { + return new TaskDeploymentDescriptor( + jobID, + jobInformation, + taskInformation, + execId, + allocationId, + indexInSubtaskGroup, + attemptNumber, + targetSlotNumber, + taskRestore, + producedResults, + inputGates); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java index 12b4277941f24..cb9126055c9a3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java @@ -33,6 +33,8 @@ import org.apache.flink.runtime.blob.VoidBlobWriter; import org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy; import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; +import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor; import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor; import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor; @@ -41,8 +43,8 @@ import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.Instance; -import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.instance.SimpleSlot; +import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; import org.apache.flink.runtime.jobgraph.DistributionPattern; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -54,19 +56,32 @@ import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings; import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway; +import org.apache.flink.runtime.jobmaster.JobMasterId; +import org.apache.flink.runtime.jobmaster.LogicalSlot; +import org.apache.flink.runtime.jobmaster.RpcTaskManagerGateway; import org.apache.flink.runtime.jobmaster.SlotOwner; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.jobmaster.TestingLogicalSlot; +import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; +import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.operators.BatchTask; +import org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway; +import org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder; import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; import org.apache.flink.runtime.taskmanager.TaskExecutionState; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testtasks.NoOpInvokable; import org.apache.flink.runtime.testutils.DirectScheduledExecutorService; +import org.apache.flink.util.FlinkException; import org.apache.flink.util.TestLogger; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; import org.junit.Test; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -74,9 +89,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.function.Function; import static junit.framework.TestCase.assertTrue; import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.getInstance; @@ -84,6 +101,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -314,7 +332,7 @@ public void testAccumulatorsAndMetricsForwarding() throws Exception { Tuple2> graphAndExecutions = setupExecution(v1, 1, v2, 1); ExecutionGraph graph = graphAndExecutions.f0; - + // verify behavior for canceled executions Execution execution1 = graphAndExecutions.f1.values().iterator().next(); @@ -322,15 +340,15 @@ public void testAccumulatorsAndMetricsForwarding() throws Exception { Map> accumulators = new HashMap<>(); accumulators.put("acc", new IntCounter(4)); AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators); - + TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics); - + graph.updateState(state); - + assertEquals(ioMetrics, execution1.getIOMetrics()); assertNotNull(execution1.getUserAccumulators()); assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue()); - + // verify behavior for failed executions Execution execution2 = graphAndExecutions.f1.values().iterator().next(); @@ -361,14 +379,14 @@ public void testAccumulatorsAndMetricsStorage() throws Exception { JobVertex v2 = new JobVertex("v2", jid2); Map executions = setupExecution(v1, 1, v2, 1).f1; - + IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); Map> accumulators = Collections.emptyMap(); Execution execution1 = executions.values().iterator().next(); execution1.cancel(); execution1.cancelingComplete(accumulators, ioMetrics); - + assertEquals(ioMetrics, execution1.getIOMetrics()); assertEquals(accumulators, execution1.getUserAccumulators()); @@ -534,7 +552,6 @@ private Tuple2> setupExecutio blobWriter, AkkaUtils.getDefaultTimeout()); checkJobOffloaded(eg); - eg.setQueuedSchedulingAllowed(false); List ordered = Arrays.asList(v1, v2); @@ -657,6 +674,112 @@ public void testEagerSchedulingWaitsOnAllInputPreferredLocations() throws Except } } + /** + * Tests that the {@link ExecutionGraph} is deployed in topological order. + */ + @Test + public void testExecutionGraphIsDeployedInTopologicalOrder() throws Exception { + final int sourceParallelism = 2; + final int sinkParallelism = 1; + + final JobVertex sourceVertex = new JobVertex("source"); + sourceVertex.setInvokableClass(NoOpInvokable.class); + sourceVertex.setParallelism(sourceParallelism); + + final JobVertex sinkVertex = new JobVertex("sink"); + sinkVertex.setInvokableClass(NoOpInvokable.class); + sinkVertex.setParallelism(sinkParallelism); + + sinkVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); + + final JobID jobId = new JobID(); + final int numberTasks = sourceParallelism + sinkParallelism; + final ArrayBlockingQueue submittedTasksQueue = new ArrayBlockingQueue<>(numberTasks); + TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder = new TestingTaskExecutorGatewayBuilder(); + testingTaskExecutorGatewayBuilder.setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> { + submittedTasksQueue.offer(taskDeploymentDescriptor.getExecutionAttemptId()); + return CompletableFuture.completedFuture(Acknowledge.get()); + }); + + final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.createTestingTaskExecutorGateway(); + final RpcTaskManagerGateway taskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, JobMasterId.generate()); + + final Collection> slotFutures = new ArrayList<>(numberTasks); + for (int i = 0; i < numberTasks; i++) { + slotFutures.add(new CompletableFuture<>()); + } + + final SlotProvider slotProvider = new IteratorTestingSlotProvider(slotFutures.iterator()); + + final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createExecutionGraph( + jobId, + slotProvider, + new NoRestartStrategy(), + new DirectScheduledExecutorService(), + sourceVertex, + sinkVertex); + executionGraph.setScheduleMode(ScheduleMode.EAGER); + executionGraph.setQueuedSchedulingAllowed(true); + + executionGraph.scheduleForExecution(); + + // change the order in which the futures are completed + final List> shuffledFutures = new ArrayList<>(slotFutures); + Collections.shuffle(shuffledFutures); + + for (CompletableFuture slotFuture : shuffledFutures) { + slotFuture.complete(new TestingLogicalSlot( + new LocalTaskManagerLocation(), + taskManagerGateway, + 0, + new AllocationID(), + new SlotRequestId(), + new SlotSharingGroupId(), + null)); + } + + final List submittedTasks = new ArrayList<>(numberTasks); + + for (int i = 0; i < numberTasks; i++) { + submittedTasks.add(submittedTasksQueue.take()); + } + + final Collection firstStage = new ArrayList<>(sourceParallelism); + for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sourceVertex.getID()).getTaskVertices()) { + firstStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId()); + } + + final Collection secondStage = new ArrayList<>(sinkParallelism); + for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sinkVertex.getID()).getTaskVertices()) { + secondStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId()); + } + + assertThat(submittedTasks, new ExecutionStageMatcher(Arrays.asList(firstStage, secondStage))); + } + + private static final class IteratorTestingSlotProvider extends TestingSlotProvider { + private IteratorTestingSlotProvider(final Iterator> slotIterator) { + super(new IteratorSlotFutureFunction(slotIterator)); + } + + private static class IteratorSlotFutureFunction implements Function> { + final Iterator> slotIterator; + + IteratorSlotFutureFunction(Iterator> slotIterator) { + this.slotIterator = slotIterator; + } + + @Override + public CompletableFuture apply(SlotRequestId slotRequestId) { + if (slotIterator.hasNext()) { + return slotIterator.next(); + } else { + return FutureUtils.completedExceptionally(new FlinkException("No more slots available.")); + } + } + } + } + private SimpleSlot createSlot(TaskManagerLocation taskManagerLocation, int index) { return new SimpleSlot( mock(SlotOwner.class), @@ -665,19 +788,6 @@ private SimpleSlot createSlot(TaskManagerLocation taskManagerLocation, int index new SimpleAckingTaskManagerGateway()); } - @SuppressWarnings("serial") - public static class FailingFinalizeJobVertex extends JobVertex { - - public FailingFinalizeJobVertex(String name, JobVertexID id) { - super(name, id); - } - - @Override - public void finalizeOnMaster(ClassLoader cl) throws Exception { - throw new Exception(); - } - } - private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception { final ScheduledExecutorService executor = TestingUtils.defaultExecutor(); @@ -715,4 +825,38 @@ private ExecutionGraph createExecutionGraph(Configuration configuration) throws timeout, LoggerFactory.getLogger(getClass())); } + + private static final class ExecutionStageMatcher extends TypeSafeMatcher> { + private final List> executionStages; + + private ExecutionStageMatcher(List> executionStages) { + this.executionStages = executionStages; + } + + @Override + protected boolean matchesSafely(List submissionOrder) { + final Iterator submissionIterator = submissionOrder.iterator(); + + for (Collection stage : executionStages) { + final Collection currentStage = new ArrayList<>(stage); + + while (!currentStage.isEmpty() && submissionIterator.hasNext()) { + if (!currentStage.remove(submissionIterator.next())) { + return false; + } + } + + if (!currentStage.isEmpty()) { + return false; + } + } + + return !submissionIterator.hasNext(); + } + + @Override + public void describeTo(Description description) { + description.appendValueList("<[", ", ", "]>", executionStages); + } + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java index 91510d1af547c..ffe1fd004812a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java @@ -973,7 +973,7 @@ private static void restartAfterFailure(ExecutionGraph eg, FiniteDuration timeou // Wait for deploying after async restart Deadline deadline = timeout.fromNow(); - waitForAllResourcesToBeAssignedAfterAsyncRestart(eg, deadline); + waitUntilAllExecutionsReachDeploying(eg, deadline); if (haltAfterRestart) { if (deadline.hasTimeLeft()) { @@ -984,6 +984,13 @@ private static void restartAfterFailure(ExecutionGraph eg, FiniteDuration timeou } } + private static void waitUntilAllExecutionsReachDeploying(ExecutionGraph eg, Deadline deadline) throws TimeoutException { + ExecutionGraphTestUtils.waitForAllExecutionsPredicate( + eg, + ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING), + deadline.timeLeft().toMillis()); + } + private static void waitForAllResourcesToBeAssignedAfterAsyncRestart(ExecutionGraph eg, Deadline deadline) throws TimeoutException { ExecutionGraphTestUtils.waitForAllExecutionsPredicate( eg, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TestingSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TestingSlotProvider.java index ed8fe130de6be..312ab110b6cc3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TestingSlotProvider.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TestingSlotProvider.java @@ -39,7 +39,7 @@ /** * {@link SlotProvider} implementation for testing purposes. */ -final class TestingSlotProvider implements SlotProvider { +public class TestingSlotProvider implements SlotProvider { private final ConcurrentMap> slotFutures; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java index 7bf9364c8d3b3..9913f36ed6c28 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.util.TestLogger; + import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderServiceTest.java new file mode 100644 index 0000000000000..e4c7cc2bf91a4 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderServiceTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.highavailability.nonha.embedded; + +import org.apache.flink.runtime.leaderelection.LeaderElectionService; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.util.TestLogger; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.hamcrest.Matchers.is; + +/** + * Tests for the {@link EmbeddedLeaderService}. + */ +public class EmbeddedLeaderServiceTest extends TestLogger { + + /** + * Tests that the {@link EmbeddedLeaderService} can handle a concurrent grant + * leadership call and a shutdown. + */ + @Test + public void testConcurrentGrantLeadershipAndShutdown() throws Exception { + final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor()); + + try { + final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService(); + + final TestingLeaderContender contender = new TestingLeaderContender(); + + leaderElectionService.start(contender); + leaderElectionService.stop(); + + try { + // check that no exception occurred + contender.getLeaderSessionFuture().get(10L, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // we haven't participated in the leader election + } + + // the election service should still be running + Assert.assertThat(embeddedLeaderService.isShutdown(), is(false)); + } finally { + embeddedLeaderService.shutdown(); + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java new file mode 100644 index 0000000000000..0b286716769ce --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.highavailability.nonha.embedded; + +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.leaderelection.LeaderContender; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +/** + * {@link LeaderContender} implementation for testing purposes. + */ +final class TestingLeaderContender implements LeaderContender { + + private final Object lock = new Object(); + + private CompletableFuture leaderSessionFuture; + + TestingLeaderContender() { + leaderSessionFuture = new CompletableFuture<>(); + } + + @Override + public void grantLeadership(UUID leaderSessionID) { + synchronized (lock) { + if (!leaderSessionFuture.isCompletedExceptionally()) { + if (!leaderSessionFuture.complete(leaderSessionID)) { + leaderSessionFuture = CompletableFuture.completedFuture(leaderSessionID); + } + } + } + } + + @Override + public void revokeLeadership() { + synchronized (lock) { + if (leaderSessionFuture.isDone() && !leaderSessionFuture.isCompletedExceptionally()) { + leaderSessionFuture = new CompletableFuture<>(); + } + } + } + + @Override + public String getAddress() { + return "foobar"; + } + + @Override + public void handleError(Exception exception) { + synchronized (lock) { + if (!(leaderSessionFuture.isCompletedExceptionally() || leaderSessionFuture.completeExceptionally(exception))) { + leaderSessionFuture = FutureUtils.completedExceptionally(exception); + } + } + } + + public void tryRethrowException() { + synchronized (lock) { + if (leaderSessionFuture.isCompletedExceptionally()) { + leaderSessionFuture.getNow(null); + } + } + } + + CompletableFuture getLeaderSessionFuture() { + synchronized (lock) { + return leaderSessionFuture; + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPoolTest.java index 537d167908f2a..a0e10d7c6872a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPoolTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPoolTest.java @@ -414,10 +414,14 @@ private BufferListener createBufferListener(int notificationTimes) { AtomicInteger times = new AtomicInteger(0); @Override - public boolean notifyBufferAvailable(Buffer buffer) { + public NotificationResult notifyBufferAvailable(Buffer buffer) { int newCount = times.incrementAndGet(); buffer.recycleBuffer(); - return newCount < notificationTimes; + if (newCount < notificationTimes) { + return NotificationResult.BUFFER_USED_NEED_MORE; + } else { + return NotificationResult.BUFFER_USED_NO_NEED_MORE; + } } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java index ec80459f0ead4..7747421fc7b5c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.io.network.ConnectionID; import org.apache.flink.runtime.io.network.ConnectionManager; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.BufferListener.NotificationResult; import org.apache.flink.runtime.io.network.buffer.BufferPool; import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool; import org.apache.flink.runtime.io.network.netty.PartitionRequestClient; @@ -150,7 +151,10 @@ private void testConcurrentReleaseAndSomething( for (int j = 0; j < 128; j++) { // this is the same buffer over and over again which will be // recycled by the RemoteInputChannel - function.apply(inputChannel, buffer.retainBuffer(), j); + Object obj = function.apply(inputChannel, buffer.retainBuffer(), j); + if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) { + buffer.recycleBuffer(); + } } if (inputChannel.isReleased()) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java index f3c72545e6950..de3f787509853 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java @@ -33,6 +33,7 @@ import org.apache.flink.core.io.InputSplitSource; import org.apache.flink.core.testutils.OneShotLatch; import org.apache.flink.queryablestate.KvStateID; +import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.blob.VoidBlobStore; import org.apache.flink.runtime.checkpoint.CheckpointProperties; @@ -84,14 +85,17 @@ import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.FlinkJobNotFoundException; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.query.KvStateLocation; import org.apache.flink.runtime.query.UnknownKvStateLocation; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; import org.apache.flink.runtime.resourcemanager.SlotRequest; import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway; +import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.TestingRpcService; +import org.apache.flink.runtime.rpc.akka.AkkaRpcService; import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation; import org.apache.flink.runtime.state.KeyGroupRange; import org.apache.flink.runtime.state.OperatorStreamStateHandle; @@ -99,6 +103,7 @@ import org.apache.flink.runtime.state.memory.ByteStreamStateHandle; import org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway; import org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder; +import org.apache.flink.runtime.taskexecutor.rpc.RpcCheckpointResponder; import org.apache.flink.runtime.taskexecutor.slot.SlotOffer; import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; import org.apache.flink.runtime.taskmanager.TaskExecutionState; @@ -106,12 +111,15 @@ import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable; import org.apache.flink.runtime.testtasks.NoOpInvokable; import org.apache.flink.runtime.util.TestingFatalErrorHandler; +import org.apache.flink.testutils.ClassLoaderUtils; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.InstantiationUtil; +import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.TestLogger; import org.apache.flink.util.function.SupplierWithException; +import akka.actor.ActorSystem; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.After; @@ -130,6 +138,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -248,6 +257,73 @@ public static void teardownClass() { } } + @Test + public void testDeclineCheckpointInvocationWithUserException() throws Exception { + RpcService rpcService1 = null; + RpcService rpcService2 = null; + try { + final ActorSystem actorSystem1 = AkkaUtils.createDefaultActorSystem(); + final ActorSystem actorSystem2 = AkkaUtils.createDefaultActorSystem(); + + rpcService1 = new AkkaRpcService(actorSystem1, testingTimeout); + rpcService2 = new AkkaRpcService(actorSystem2, testingTimeout); + + final CompletableFuture declineCheckpointMessageFuture = new CompletableFuture<>(); + + final JobManagerSharedServices jobManagerSharedServices = new TestingJobManagerSharedServicesBuilder().build(); + final JobMasterConfiguration jobMasterConfiguration = JobMasterConfiguration.fromConfiguration(configuration); + final JobMaster jobMaster = new JobMaster( + rpcService1, + jobMasterConfiguration, + jmResourceId, + jobGraph, + haServices, + DefaultSlotPoolFactory.fromConfiguration(configuration, rpcService1), + jobManagerSharedServices, + heartbeatServices, + blobServer, + UnregisteredJobManagerJobMetricGroupFactory.INSTANCE, + new NoOpOnCompletionActions(), + testingFatalErrorHandler, + JobMasterTest.class.getClassLoader()) { + @Override + public void declineCheckpoint(DeclineCheckpoint declineCheckpoint) { + declineCheckpointMessageFuture.complete(declineCheckpoint.getReason()); + } + }; + + jobMaster.start(jobMasterId, testingTimeout).get(); + + final String className = "UserException"; + final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava( + temporaryFolder.newFolder(), + className + ".java", + String.format("public class %s extends RuntimeException { public %s() {super(\"UserMessage\");} }", + className, + className)); + + Throwable userException = (Throwable) Class.forName(className, false, userClassLoader).newInstance(); + + JobMasterGateway jobMasterGateway = + rpcService2.connect(jobMaster.getAddress(), jobMaster.getFencingToken(), JobMasterGateway.class).get(); + + RpcCheckpointResponder rpcCheckpointResponder = new RpcCheckpointResponder(jobMasterGateway); + rpcCheckpointResponder.declineCheckpoint( + jobGraph.getJobID(), + new ExecutionAttemptID(1, 1), + 1, + userException + ); + + Throwable throwable = declineCheckpointMessageFuture.get(testingTimeout.toMilliseconds(), + TimeUnit.MILLISECONDS); + assertThat(throwable, instanceOf(SerializedThrowable.class)); + assertThat(throwable.getMessage(), equalTo(userException.getMessage())); + } finally { + RpcUtils.terminateRpcServices(testingTimeout, rpcService1, rpcService2); + } + } + @Test public void testHeartbeatTimeoutWithTaskManager() throws Exception { final CompletableFuture heartbeatResourceIdFuture = new CompletableFuture<>(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java index c9c55a1da8ff6..f5f7f8e3415c3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java @@ -41,6 +41,7 @@ import org.apache.flink.runtime.jobmaster.SerializedInputSplit; import org.apache.flink.runtime.jobmaster.message.ClassloadingProps; import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.messages.webmonitor.JobDetails; import org.apache.flink.runtime.query.KvStateLocation; import org.apache.flink.runtime.registration.RegistrationResponse; @@ -144,7 +145,7 @@ public class TestingJobMasterGateway implements JobMasterGateway { private final Consumer> acknowledgeCheckpointConsumer; @Nonnull - private final Consumer> declineCheckpointConsumer; + private final Consumer declineCheckpointConsumer; @Nonnull private final Supplier fencingTokenSupplier; @@ -183,7 +184,7 @@ public TestingJobMasterGateway( @Nonnull Function> requestOperatorBackPressureStatsFunction, @Nonnull BiConsumer notifyAllocationFailureConsumer, @Nonnull Consumer> acknowledgeCheckpointConsumer, - @Nonnull Consumer> declineCheckpointConsumer, + @Nonnull Consumer declineCheckpointConsumer, @Nonnull Supplier fencingTokenSupplier, @Nonnull BiFunction> requestKvStateLocationFunction, @Nonnull Function, CompletableFuture> notifyKvStateRegisteredFunction, @@ -335,8 +336,8 @@ public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttem } @Override - public void declineCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, Throwable cause) { - declineCheckpointConsumer.accept(Tuple4.of(jobID, executionAttemptID, checkpointId, cause)); + public void declineCheckpoint(DeclineCheckpoint declineCheckpoint) { + declineCheckpointConsumer.accept(declineCheckpoint); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java index e40b752f24842..b52df9ee05233 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java @@ -40,6 +40,7 @@ import org.apache.flink.runtime.jobmaster.SerializedInputSplit; import org.apache.flink.runtime.jobmaster.message.ClassloadingProps; import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.messages.webmonitor.JobDetails; import org.apache.flink.runtime.query.KvStateLocation; import org.apache.flink.runtime.query.UnknownKvStateLocation; @@ -96,7 +97,7 @@ public class TestingJobMasterGatewayBuilder { private Function> requestOperatorBackPressureStatsFunction = ignored -> CompletableFuture.completedFuture(OperatorBackPressureStatsResponse.of(null)); private BiConsumer notifyAllocationFailureConsumer = (ignoredA, ignoredB) -> {}; private Consumer> acknowledgeCheckpointConsumer = ignored -> {}; - private Consumer> declineCheckpointConsumer = ignored -> {}; + private Consumer declineCheckpointConsumer = ignored -> {}; private Supplier fencingTokenSupplier = () -> JOB_MASTER_ID; private BiFunction> requestKvStateLocationFunction = (ignoredA, registrationName) -> FutureUtils.completedExceptionally(new UnknownKvStateLocation(registrationName)); private Function, CompletableFuture> notifyKvStateRegisteredFunction = ignored -> CompletableFuture.completedFuture(Acknowledge.get()); @@ -222,7 +223,7 @@ public TestingJobMasterGatewayBuilder setAcknowledgeCheckpointConsumer(Consumer< return this; } - public TestingJobMasterGatewayBuilder setDeclineCheckpointConsumer(Consumer> declineCheckpointConsumer) { + public TestingJobMasterGatewayBuilder setDeclineCheckpointConsumer(Consumer declineCheckpointConsumer) { this.declineCheckpointConsumer = declineCheckpointConsumer; return this; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java index 5f83e794ff986..1aab6f7de43ea 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java @@ -70,7 +70,10 @@ public Object getValue() { Collections.>emptyMap()); // no metrics should be serialized - Assert.assertEquals(0, output.serializedMetrics.length); + Assert.assertEquals(0, output.serializedCounters.length); + Assert.assertEquals(0, output.serializedGauges.length); + Assert.assertEquals(0, output.serializedHistograms.length); + Assert.assertEquals(0, output.serializedMeters.length); List deserialized = deserializer.deserialize(output); Assert.assertEquals(0, deserialized.size()); @@ -141,7 +144,8 @@ public long getCount() { gauges.put(g1, new Tuple2(new QueryScopeInfo.TaskQueryScopeInfo("jid", "vid", 2, "D"), "g1")); histograms.put(h1, new Tuple2(new QueryScopeInfo.OperatorQueryScopeInfo("jid", "vid", 2, "opname", "E"), "h1")); - MetricDumpSerialization.MetricSerializationResult serialized = serializer.serialize(counters, gauges, histograms, meters); + MetricDumpSerialization.MetricSerializationResult serialized = serializer.serialize( + counters, gauges, histograms, meters); List deserialized = deserializer.deserialize(serialized); // ===== Counters ============================================================================================== diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java index 3767421b7d62f..ccc2236fa8499 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.metrics.dump; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Gauge; @@ -38,6 +39,10 @@ import akka.testkit.TestActorRef; import org.junit.Test; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -47,9 +52,8 @@ public class MetricQueryServiceTest extends TestLogger { @Test public void testCreateDump() throws Exception { - ActorSystem s = AkkaUtils.createLocalActorSystem(new Configuration()); - ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null); + ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null, Long.MAX_VALUE); TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class)); TestActor testActor = (TestActor) testActorRef.underlyingActor(); @@ -98,7 +102,10 @@ public long getCount() { MetricDumpSerialization.MetricSerializationResult dump = (MetricDumpSerialization.MetricSerializationResult) testActor.message; testActor.message = null; - assertTrue(dump.serializedMetrics.length > 0); + assertTrue(dump.serializedCounters.length > 0); + assertTrue(dump.serializedGauges.length > 0); + assertTrue(dump.serializedHistograms.length > 0); + assertTrue(dump.serializedMeters.length > 0); MetricQueryService.notifyOfRemovedMetric(serviceActor, c); MetricQueryService.notifyOfRemovedMetric(serviceActor, g); @@ -114,7 +121,103 @@ public long getCount() { MetricDumpSerialization.MetricSerializationResult emptyDump = (MetricDumpSerialization.MetricSerializationResult) testActor.message; testActor.message = null; - assertEquals(0, emptyDump.serializedMetrics.length); + assertEquals(0, emptyDump.serializedCounters.length); + assertEquals(0, emptyDump.serializedGauges.length); + assertEquals(0, emptyDump.serializedHistograms.length); + assertEquals(0, emptyDump.serializedMeters.length); + + s.terminate(); + } + + @Test + public void testHandleOversizedMetricMessage() throws Exception { + ActorSystem s = AkkaUtils.createLocalActorSystem(new Configuration()); + final long sizeLimit = 200L; + ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null, sizeLimit); + TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class)); + TestActor testActor = (TestActor) testActorRef.underlyingActor(); + + final Counter c = new SimpleCounter(); + final Histogram h = new TestHistogram(); + final Meter m = new Meter() { + + @Override + public void markEvent() { + } + + @Override + public void markEvent(long n) { + } + + @Override + public double getRate() { + return 5; + } + + @Override + public long getCount() { + return 10; + } + }; + + MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()); + final TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id"); + + final String gaugeValue = "Hello"; + final long requiredGaugesToExceedLimit = sizeLimit / gaugeValue.length() + 1; + List>> gauges = LongStream.range(0, requiredGaugesToExceedLimit) + .mapToObj(x -> Tuple2.of("gauge" + x, (Gauge) () -> "Hello" + x)) + .collect(Collectors.toList()); + gauges.forEach(gauge -> MetricQueryService.notifyOfAddedMetric(serviceActor, gauge.f1, gauge.f0, tm)); + + MetricQueryService.notifyOfAddedMetric(serviceActor, c, "counter", tm); + MetricQueryService.notifyOfAddedMetric(serviceActor, h, "histogram", tm); + MetricQueryService.notifyOfAddedMetric(serviceActor, m, "meter", tm); + + serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef); + synchronized (testActor.lock) { + if (testActor.message == null) { + testActor.lock.wait(); + } + } + + MetricDumpSerialization.MetricSerializationResult dump = (MetricDumpSerialization.MetricSerializationResult) testActor.message; + testActor.message = null; + assertTrue(dump.serializedCounters.length > 0); + assertEquals(1, dump.numCounters); + assertTrue(dump.serializedMeters.length > 0); + assertEquals(1, dump.numMeters); + + // gauges exceeded the size limit and will be excluded + assertEquals(0, dump.serializedGauges.length); + assertEquals(0, dump.numGauges); + + assertTrue(dump.serializedHistograms.length > 0); + assertEquals(1, dump.numHistograms); + + // unregister all but one gauge to ensure gauges are reported again if the remaining fit + for (int x = 1; x < gauges.size(); x++) { + MetricQueryService.notifyOfRemovedMetric(serviceActor, gauges.get(x).f1); + } + + serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef); + synchronized (testActor.lock) { + if (testActor.message == null) { + testActor.lock.wait(); + } + } + + MetricDumpSerialization.MetricSerializationResult recoveredDump = (MetricDumpSerialization.MetricSerializationResult) testActor.message; + testActor.message = null; + + assertTrue(recoveredDump.serializedCounters.length > 0); + assertEquals(1, recoveredDump.numCounters); + assertTrue(recoveredDump.serializedMeters.length > 0); + assertEquals(1, recoveredDump.numMeters); + assertTrue(recoveredDump.serializedGauges.length > 0); + assertEquals(1, recoveredDump.numGauges); + assertTrue(recoveredDump.serializedHistograms.length > 0); + assertEquals(1, recoveredDump.numHistograms); s.terminate(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java index f8ed3c6a6d8e0..d750f63cfd869 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java @@ -21,17 +21,22 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MetricOptions; +import org.apache.flink.core.testutils.BlockerSync; import org.apache.flink.metrics.CharacterFilter; import org.apache.flink.metrics.Metric; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.reporter.MetricReporter; +import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; import org.apache.flink.runtime.metrics.MetricRegistryImpl; import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; +import org.apache.flink.runtime.metrics.scope.ScopeFormats; import org.apache.flink.runtime.metrics.util.TestReporter; import org.junit.Test; +import javax.annotation.Nullable; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -105,6 +110,30 @@ public void testScopeCachingForMultipleReporters() throws Exception { } } + @Test + public void testLogicalScopeCachingForMultipleReporters() throws Exception { + Configuration config = new Configuration(); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, LogicalScopeReporter1.class.getName()); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, LogicalScopeReporter2.class.getName()); + + MetricRegistryImpl testRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config)); + try { + MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id") + .addGroup("B") + .addGroup("C"); + tmGroup.counter("1"); + assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size()); + for (MetricReporter reporter : testRegistry.getReporters()) { + ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter; + if (typedReporter.failureCause != null) { + throw typedReporter.failureCause; + } + } + } finally { + testRegistry.shutdown().get(); + } + } + private abstract static class ScopeCheckingTestReporter extends TestReporter { protected Exception failureCause; @@ -175,6 +204,38 @@ public String filterCharacters(String input) { } } + /** + * Reporter that verifies the logical-scope caching behavior. + */ + public static final class LogicalScopeReporter1 extends ScopeCheckingTestReporter { + @Override + public String filterCharacters(String input) { + return FILTER_B.filterCharacters(input); + } + + @Override + public void checkScopes(Metric metric, String metricName, MetricGroup group) { + final String logicalScope = ((FrontMetricGroup>) group).getLogicalScope(this, '-'); + assertEquals("taskmanager-X-C", logicalScope); + } + } + + /** + * Reporter that verifies the logical-scope caching behavior. + */ + public static final class LogicalScopeReporter2 extends ScopeCheckingTestReporter { + @Override + public String filterCharacters(String input) { + return FILTER_C.filterCharacters(input); + } + + @Override + public void checkScopes(Metric metric, String metricName, MetricGroup group) { + final String logicalScope = ((FrontMetricGroup>) group).getLogicalScope(this, ','); + assertEquals("taskmanager,B,X", logicalScope); + } + } + @Test public void testScopeGenerationWithoutReporters() throws Exception { Configuration config = new Configuration(); @@ -196,4 +257,89 @@ public void testScopeGenerationWithoutReporters() throws Exception { testRegistry.shutdown().get(); } } + + @Test + public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException { + final TestMetricRegistry registry = new TestMetricRegistry(); + + final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent"); + final MetricGroup child = parent.addGroup("child"); + + final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter")); + final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter")); + + final BlockerSync parentSync = new BlockerSync(); + final BlockerSync childSync = new BlockerSync(); + + try { + // start both threads and have them block in the registry, so they acquire the lock of their respective group + registry.setOnRegistrationAction(childSync::blockNonInterruptible); + childRegisteringThread.start(); + childSync.awaitBlocker(); + + registry.setOnRegistrationAction(parentSync::blockNonInterruptible); + parentRegisteringThread.start(); + parentSync.awaitBlocker(); + + // the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter + // the child thread continues execution and calls getAllVariables() + // in the past this would block indefinitely since the method acquires the locks of all parent groups + childSync.releaseBlocker(); + // wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent + childRegisteringThread.join(1000 * 10); + + parentSync.releaseBlocker(); + parentRegisteringThread.join(); + } finally { + parentSync.releaseBlocker(); + childSync.releaseBlocker(); + parentRegisteringThread.join(); + childRegisteringThread.join(); + } + } + + private static final class TestMetricRegistry implements MetricRegistry { + + private Runnable onRegistrationAction; + + void setOnRegistrationAction(Runnable onRegistrationAction) { + this.onRegistrationAction = onRegistrationAction; + } + + @Override + public char getDelimiter() { + return 0; + } + + @Override + public char getDelimiter(int index) { + return 0; + } + + @Override + public int getNumberReporters() { + return 0; + } + + @Override + public void register(Metric metric, String metricName, AbstractMetricGroup group) { + onRegistrationAction.run(); + group.getAllVariables(); + } + + @Override + public void unregister(Metric metric, String metricName, AbstractMetricGroup group) { + } + + @Override + public ScopeFormats getScopeFormats() { + return null; + } + + @Nullable + @Override + public String getMetricQueryServicePath() { + return null; + } + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java index 71ae7f18ecb7e..41d3fc9cb2bef 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java @@ -19,6 +19,8 @@ package org.apache.flink.runtime.metrics.groups; import org.apache.flink.api.common.JobID; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.CharacterFilter; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.Metric; @@ -31,6 +33,7 @@ import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; import org.apache.flink.runtime.metrics.scope.ScopeFormat; import org.apache.flink.runtime.metrics.util.DummyCharacterFilter; +import org.apache.flink.runtime.metrics.util.TestReporter; import org.apache.flink.util.AbstractID; import org.apache.flink.util.TestLogger; @@ -38,10 +41,13 @@ import org.junit.Before; import org.junit.Test; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -216,6 +222,33 @@ public void testNameCollisionAfterKeyValueGroup() { assertFalse("Value is present in logical scope.", logicalScope.contains(value)); } + /** + * Verifies that calling {@link AbstractMetricGroup#getLogicalScope(CharacterFilter, char, int)} on {@link GenericValueMetricGroup} + * should ignore value as well. + */ + @Test + public void testLogicalScopeShouldIgnoreValueGroupName() throws Exception { + Configuration config = new Configuration(); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName()); + + MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config)); + try { + GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root"); + + String key = "key"; + String value = "value"; + + MetricGroup group = root.addGroup(key, value); + + String logicalScope = ((AbstractMetricGroup) group) + .getLogicalScope(new DummyCharacterFilter(), registry.getDelimiter(), 0); + assertThat("Key is missing from logical scope.", logicalScope, containsString(key)); + assertThat("Value is present in logical scope.", logicalScope, not(containsString(value))); + } finally { + registry.shutdown().get(); + } + } + @Test public void closedGroupDoesNotRegisterMetrics() { GenericMetricGroup group = new GenericMetricGroup( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/util/MetricUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/util/MetricUtilsTest.java index bea0f2a2153f5..3edb711c351a5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/util/MetricUtilsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/util/MetricUtilsTest.java @@ -20,10 +20,13 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MetricOptions; +import org.apache.flink.metrics.Gauge; import org.apache.flink.runtime.akka.AkkaUtils; +import org.apache.flink.runtime.metrics.MetricNames; import org.apache.flink.util.TestLogger; import akka.actor.ActorSystem; +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,4 +60,56 @@ public void testStartMetricActorSystemRespectsThreadPriority() throws Exception AkkaUtils.terminateActorSystem(actorSystem).get(); } } + + @Test + public void testNonHeapMetricsCompleteness() { + final InterceptingOperatorMetricGroup nonHeapMetrics = new InterceptingOperatorMetricGroup(); + + MetricUtils.instantiateNonHeapMemoryMetrics(nonHeapMetrics); + + Assert.assertNotNull(nonHeapMetrics.get(MetricNames.MEMORY_USED)); + Assert.assertNotNull(nonHeapMetrics.get(MetricNames.MEMORY_COMMITTED)); + Assert.assertNotNull(nonHeapMetrics.get(MetricNames.MEMORY_MAX)); + } + + @Test + public void testHeapMetricsCompleteness() { + final InterceptingOperatorMetricGroup heapMetrics = new InterceptingOperatorMetricGroup(); + + MetricUtils.instantiateHeapMemoryMetrics(heapMetrics); + + Assert.assertNotNull(heapMetrics.get(MetricNames.MEMORY_USED)); + Assert.assertNotNull(heapMetrics.get(MetricNames.MEMORY_COMMITTED)); + Assert.assertNotNull(heapMetrics.get(MetricNames.MEMORY_MAX)); + } + + /** + * Tests that heap/non-heap metrics do not rely on a static MemoryUsage instance. + * + *

    We can only check this easily for the currently used heap memory, so we use it this as a proxy for testing + * the functionality in general. + */ + @Test + public void testHeapMetrics() throws Exception { + final InterceptingOperatorMetricGroup heapMetrics = new InterceptingOperatorMetricGroup(); + + MetricUtils.instantiateHeapMemoryMetrics(heapMetrics); + + @SuppressWarnings("unchecked") + final Gauge used = (Gauge) heapMetrics.get(MetricNames.MEMORY_USED); + + final long usedHeapInitially = used.getValue(); + + // check memory usage difference multiple times since other tests may affect memory usage as well + for (int x = 0; x < 10; x++) { + final byte[] array = new byte[1024 * 1024 * 8]; + final long usedHeapAfterAllocation = used.getValue(); + + if (usedHeapInitially != usedHeapAfterAllocation) { + return; + } + Thread.sleep(50); + } + Assert.fail("Heap usage metric never changed it's value."); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java index c108eaee36a38..585e1badf473c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java @@ -101,7 +101,42 @@ public void runJobWithMultipleRpcServices() throws Exception { } @Test - public void testHandleJobsWhenNotEnoughSlot() throws Exception { + public void testHandleStreamingJobsWhenNotEnoughSlot() throws Exception { + try { + setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode.EAGER); + fail("Job should fail."); + } catch (JobExecutionException e) { + assertTrue(findThrowableWithMessage(e, "Job execution failed.").isPresent()); + assertTrue(findThrowable(e, NoResourceAvailableException.class).isPresent()); + assertTrue(findThrowableWithMessage(e, "Slots required: 2, slots allocated: 1").isPresent()); + } + } + + @Test + public void testHandleBatchJobsWhenNotEnoughSlot() throws Exception { + try { + setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode.LAZY_FROM_SOURCES); + fail("Job should fail."); + } catch (JobExecutionException e) { + assertTrue(findThrowableWithMessage(e, "Job execution failed.").isPresent()); + assertTrue(findThrowable(e, NoResourceAvailableException.class).isPresent()); + assertTrue(findThrowableWithMessage(e, "Could not allocate enough slots").isPresent()); + } + } + + private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception { + final JobVertex vertex = new JobVertex("Test Vertex"); + vertex.setParallelism(2); + vertex.setMaxParallelism(2); + vertex.setInvokableClass(BlockingNoOpInvokable.class); + + final JobGraph jobGraph = new JobGraph("Test Job", vertex); + jobGraph.setScheduleMode(scheduleMode); + + runHandleJobsWhenNotEnoughSlots(jobGraph); + } + + private void runHandleJobsWhenNotEnoughSlots(final JobGraph jobGraph) throws Exception { final Configuration configuration = getDefaultConfiguration(); configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 100L); @@ -114,24 +149,7 @@ public void testHandleJobsWhenNotEnoughSlot() throws Exception { try (final MiniCluster miniCluster = new MiniCluster(cfg)) { miniCluster.start(); - final JobVertex vertex = new JobVertex("Test Vertex"); - vertex.setParallelism(2); - vertex.setMaxParallelism(2); - vertex.setInvokableClass(BlockingNoOpInvokable.class); - - final JobGraph jobGraph = new JobGraph("Test Job", vertex); - jobGraph.setScheduleMode(ScheduleMode.EAGER); - - try { - miniCluster.executeJobBlocking(jobGraph); - - fail("Job should fail."); - } catch (JobExecutionException e) { - assertTrue(findThrowableWithMessage(e, "Job execution failed.").isPresent()); - - assertTrue(findThrowable(e, NoResourceAvailableException.class).isPresent()); - assertTrue(findThrowableWithMessage(e, "Slots required: 2, slots allocated: 1").isPresent()); - } + miniCluster.executeJobBlocking(jobGraph); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java index 858c6620b3c54..771fd8a837c2d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java @@ -20,6 +20,7 @@ import org.apache.flink.runtime.io.network.netty.NettyLeakDetectionResource; import org.apache.flink.runtime.rest.util.RestMapperUtils; +import org.apache.flink.util.FileUtils; import org.apache.flink.util.TestLogger; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -123,6 +124,20 @@ private static MultipartBody.Builder addJsonPart(MultipartBody.Builder builder, return builder.addFormDataPart(attribute, jsonPayload); } + @Test + public void testUploadDirectoryRegeneration() throws Exception { + OkHttpClient client = new OkHttpClient(); + + MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler(); + + FileUtils.deleteDirectory(MULTIPART_UPLOAD_RESOURCE.getUploadDirectory().toFile()); + + Request fileRequest = buildFileRequest(fileHandler.getMessageHeaders().getTargetRestEndpointURL()); + try (Response response = client.newCall(fileRequest).execute()) { + assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code()); + } + } + @Test public void testMixedMultipart() throws Exception { OkHttpClient client = new OkHttpClient(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java index 22de8a1dcde4d..65690c80b09e6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java @@ -163,6 +163,10 @@ public MultipartJsonHandler getJsonHandler() { return jsonHandler; } + public Path getUploadDirectory() { + return configuredUploadDir; + } + public void resetState() { mixedHandler.lastReceivedRequest = null; jsonHandler.lastReceivedRequest = null; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java index da8182a8edea5..61c028ffab531 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java @@ -99,7 +99,7 @@ public void testUpdate() throws Exception { MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID); when(jmQueryService.queryMetrics(any(Time.class))) - .thenReturn(CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], 0, 0, 0, 0))); + .thenReturn(CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0))); when(tmQueryService.queryMetrics(any(Time.class))) .thenReturn(CompletableFuture.completedFuture(requestMetricsAnswer)); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java index c1f08e06b3c9f..55aacb230574c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java @@ -251,7 +251,7 @@ private void assertEqualStateMetaInfoSnapshots(StateMetaInfoSnapshot expected, S Assert.assertEquals(expected.getBackendStateType(), actual.getBackendStateType()); Assert.assertEquals(expected.getOptionsImmutable(), actual.getOptionsImmutable()); Assert.assertEquals( - expected.getSerializerConfigSnapshotsImmutable(), - actual.getSerializerConfigSnapshotsImmutable()); + expected.getSerializerSnapshotsImmutable(), + actual.getSerializerSnapshotsImmutable()); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java index f5f30d5037a4b..5511792673e9b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java @@ -34,21 +34,20 @@ import org.apache.flink.runtime.checkpoint.StateObjectCollection; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.operators.testutils.DummyEnvironment; -import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement; +import org.apache.flink.runtime.testutils.statemigration.TestType; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.StateMigrationException; import org.apache.flink.util.TestLogger; + import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Objects; import java.util.concurrent.RunnableFuture; /** @@ -69,17 +68,6 @@ public abstract class StateBackendMigrationTestBase kvId = new ValueStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ValueState valueState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -113,10 +101,10 @@ public void testKeyedValueStateMigration() throws Exception { backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - // the new serializer is REQUIRES_MIGRATION, and has a completely new serialization schema. + // the new serializer is V2, and has a completely new serialization schema. kvId = new ValueStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.V2TestTypeSerializer()); valueState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -151,7 +139,7 @@ public void testKeyedListStateMigration() throws Exception { try { ListStateDescriptor kvId = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState listState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -174,10 +162,10 @@ public void testKeyedListStateMigration() throws Exception { backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - // the new serializer is REQUIRES_MIGRATION, and has a completely new serialization schema. + // the new serializer is V2, and has a completely new serialization schema. kvId = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.V2TestTypeSerializer()); listState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -221,7 +209,7 @@ public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatib try { ValueStateDescriptor kvId = new ValueStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ValueState valueState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -241,10 +229,12 @@ public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatib kvId = new ValueStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE)); + new TestType.IncompatibleTestTypeSerializer()); // the new serializer is INCOMPATIBLE, so registering the state should fail backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); }finally { @@ -263,7 +253,7 @@ public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatibl try { ListStateDescriptor kvId = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState listState = backend .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); @@ -288,10 +278,12 @@ public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatibl kvId = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.IncompatibleTestTypeSerializer()); // the new serializer is INCOMPATIBLE, so registering the state should fail backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } finally { @@ -308,7 +300,7 @@ public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() try { InternalPriorityQueue internalPriorityQueue = backend.create( - "testPriorityQueue", new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + "testPriorityQueue", new TestType.V1TestTypeSerializer()); internalPriorityQueue.add(new TestType("key-1", 123)); internalPriorityQueue.add(new TestType("key-2", 346)); @@ -321,7 +313,7 @@ public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); backend.create( - "testPriorityQueue", new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE)); + "testPriorityQueue", new TestType.IncompatibleTestTypeSerializer()); Assert.fail("should have failed"); } catch (Exception e) { @@ -337,7 +329,7 @@ public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() thr SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); AbstractKeyedStateBackend backend = createKeyedBackend( - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); final String stateName = "test-name"; try { @@ -357,14 +349,18 @@ public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() thr try { // the new key serializer is incompatible; this should fail the restore - restoreKeyedBackend(new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE), snapshot); + restoreKeyedBackend(new TestType.IncompatibleTestTypeSerializer(), snapshot); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } try { // the new key serializer requires migration; this should fail the restore - restoreKeyedBackend(new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION), snapshot); + restoreKeyedBackend(new TestType.V2TestTypeSerializer(), snapshot); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } @@ -386,7 +382,7 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib ValueState valueState = backend .getPartitionedState( new TestType("namespace", 123), - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS), + new TestType.V1TestTypeSerializer(), kvId); backend.setCurrentKey(1); @@ -397,26 +393,33 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib KeyedStateHandle snapshot = runSnapshot( backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry); - backend.dispose(); + // test incompatible namespace serializer; start with a freshly restored backend + backend.dispose(); backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - try { // the new namespace serializer is incompatible; this should fail the restore backend.getPartitionedState( new TestType("namespace", 123), - new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE), + new TestType.IncompatibleTestTypeSerializer(), kvId); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } + // test namespace serializer that requires migration; start with a freshly restored backend + backend.dispose(); + backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); try { // the new namespace serializer requires migration; this should fail the restore backend.getPartitionedState( new TestType("namespace", 123), - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION), + new TestType.V2TestTypeSerializer(), kvId); + + Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } @@ -439,7 +442,7 @@ public void testOperatorParitionableListStateMigration() throws Exception { try { ListStateDescriptor descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState state = backend.getListState(descriptor); state.add(new TestType("foo", 13)); @@ -453,7 +456,7 @@ public void testOperatorParitionableListStateMigration() throws Exception { descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.V2TestTypeSerializer()); state = backend.getListState(descriptor); // the state backend should have decided whether or not it needs to perform state migration; @@ -478,7 +481,7 @@ public void testUnionListStateMigration() throws Exception { try { ListStateDescriptor descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState state = backend.getUnionListState(descriptor); state.add(new TestType("foo", 13)); @@ -492,7 +495,7 @@ public void testUnionListStateMigration() throws Exception { descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.V2TestTypeSerializer()); state = backend.getUnionListState(descriptor); // the state backend should have decided whether or not it needs to perform state migration; @@ -518,7 +521,7 @@ public void testBroadcastStateValueMigration() throws Exception { MapStateDescriptor descriptor = new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); BroadcastState state = backend.getBroadcastState(descriptor); state.put(3, new TestType("foo", 13)); @@ -533,7 +536,7 @@ public void testBroadcastStateValueMigration() throws Exception { descriptor = new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.V2TestTypeSerializer()); state = backend.getBroadcastState(descriptor); // the state backend should have decided whether or not it needs to perform state migration; @@ -556,7 +559,7 @@ public void testBroadcastStateKeyMigration() throws Exception { try { MapStateDescriptor descriptor = new MapStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS), + new TestType.V1TestTypeSerializer(), IntSerializer.INSTANCE); BroadcastState state = backend.getBroadcastState(descriptor); @@ -571,7 +574,7 @@ public void testBroadcastStateKeyMigration() throws Exception { descriptor = new MapStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION), + new TestType.V2TestTypeSerializer(), IntSerializer.INSTANCE); state = backend.getBroadcastState(descriptor); @@ -595,7 +598,7 @@ public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsI try { ListStateDescriptor descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState state = backend.getListState(descriptor); state.add(new TestType("foo", 13)); @@ -609,7 +612,7 @@ public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsI descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE)); + new TestType.IncompatibleTestTypeSerializer()); // the new serializer is INCOMPATIBLE, so registering the state should fail backend.getListState(descriptor); @@ -632,7 +635,7 @@ public void testUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() t try { ListStateDescriptor descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); ListState state = backend.getUnionListState(descriptor); state.add(new TestType("foo", 13)); @@ -646,7 +649,7 @@ public void testUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() t descriptor = new ListStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE)); + new TestType.IncompatibleTestTypeSerializer()); // the new serializer is INCOMPATIBLE, so registering the state should fail backend.getUnionListState(descriptor); @@ -670,7 +673,7 @@ public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatibl MapStateDescriptor descriptor = new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS)); + new TestType.V1TestTypeSerializer()); BroadcastState state = backend.getBroadcastState(descriptor); state.put(3, new TestType("foo", 13)); @@ -685,10 +688,12 @@ public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatibl descriptor = new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION)); + new TestType.IncompatibleTestTypeSerializer()); // the new value serializer is INCOMPATIBLE, so registering the state should fail backend.getBroadcastState(descriptor); + + Assert.fail("should have failed."); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } finally { @@ -706,7 +711,7 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible( try { MapStateDescriptor descriptor = new MapStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS), + new TestType.V1TestTypeSerializer(), IntSerializer.INSTANCE); BroadcastState state = backend.getBroadcastState(descriptor); @@ -721,11 +726,13 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible( descriptor = new MapStateDescriptor<>( stateName, - new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE), + new TestType.IncompatibleTestTypeSerializer(), IntSerializer.INSTANCE); // the new key serializer is INCOMPATIBLE, so registering the state should fail backend.getBroadcastState(descriptor); + + Assert.fail("should have failed."); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); } finally { @@ -737,223 +744,6 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible( // Test types, serializers, and serializer snapshots // ------------------------------------------------------------------------------- - /** - * The type used as state under tests. - * - *

    This is implemented so that the type can also be used as keyed priority queue state. - */ - private static class TestType implements HeapPriorityQueueElement, PriorityComparable, Keyed { - - private int index; - - private final int value; - private final String key; - - public TestType(String key, int value) { - this.key = key; - this.value = value; - } - - @Override - public String getKey() { - return key; - } - - @Override - public int comparePriorityTo(@Nonnull TestType other) { - return Integer.compare(value, other.value); - } - - @Override - public int getInternalIndex() { - return index; - } - - @Override - public void setInternalIndex(int newIndex) { - this.index = newIndex; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || !(obj instanceof StateBackendMigrationTestBase.TestType)) { - return false; - } - - if (obj == this) { - return true; - } - - TestType other = (TestType) obj; - return Objects.equals(key, other.key) && value == other.value; - } - - @Override - public int hashCode() { - return 31 * key.hashCode() + value; - } - } - - private static class TestSerializer extends TypeSerializer { - - private static final String MIGRATION_PAYLOAD = "random-migration-payload"; - - private final SerializerCompatibilityType compatibilityType; - - TestSerializer(SerializerCompatibilityType compatibilityType) { - this.compatibilityType = compatibilityType; - } - - // -------------------------------------------------------------------------------- - // State serialization relevant methods - // -------------------------------------------------------------------------------- - - @Override - public void serialize(TestType record, DataOutputView target) throws IOException { - switch (compatibilityType) { - case COMPATIBLE_AS_IS: - target.writeUTF(record.getKey()); - target.writeInt(record.value); - break; - - case REQUIRES_MIGRATION: - target.writeUTF(record.getKey()); - target.writeUTF(MIGRATION_PAYLOAD); - target.writeInt(record.value); - target.writeBoolean(true); - break; - - case INCOMPATIBLE: - // the serializer shouldn't be used in this case - throw new UnsupportedOperationException(); - } - } - - @Override - public TestType deserialize(DataInputView source) throws IOException { - String key; - int value; - - switch (compatibilityType) { - case COMPATIBLE_AS_IS: - key = source.readUTF(); - value = source.readInt(); - break; - - case REQUIRES_MIGRATION: - key = source.readUTF(); - Assert.assertEquals(MIGRATION_PAYLOAD, source.readUTF()); - value = source.readInt(); - Assert.assertTrue(source.readBoolean()); - break; - - default: - case INCOMPATIBLE: - // the serializer shouldn't be used in this case - throw new UnsupportedOperationException(); - } - - return new TestType(key, value); - } - - @Override - public TestType copy(TestType from) { - return new TestType(from.key, from.value); - } - - @Override - public TypeSerializerSnapshot snapshotConfiguration() { - return new TestSerializerSnapshot(); - } - - // -------------------------------------------------------------------------------- - // Miscellaneous serializer methods - // -------------------------------------------------------------------------------- - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - serialize(deserialize(source), target); - } - - @Override - public TestType deserialize(TestType reuse, DataInputView source) throws IOException { - return deserialize(source); - } - - @Override - public TestType copy(TestType from, TestType reuse) { - return copy(from); - } - - @Override - public TestType createInstance() { - throw new UnsupportedOperationException(); - } - - @Override - public TypeSerializer duplicate() { - return this; - } - - @Override - public boolean isImmutableType() { - return false; - } - - @Override - public int getLength() { - return -1; - } - - @Override - public boolean canEqual(Object obj) { - return getClass().equals(obj.getClass()); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } - - @Override - public boolean equals(Object obj) { - return obj == this; - } - } - - public static class TestSerializerSnapshot implements TypeSerializerSnapshot { - - @Override - public int getCurrentVersion() { - return 1; - } - - @Override - public TypeSerializer restoreSerializer() { - return new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS); - } - - @Override - public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { - switch (((TestSerializer) newSerializer).compatibilityType) { - case COMPATIBLE_AS_IS: - return TypeSerializerSchemaCompatibility.compatibleAsIs(); - case REQUIRES_MIGRATION: - return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); - case INCOMPATIBLE: - return TypeSerializerSchemaCompatibility.incompatible(); - default: - throw new UnsupportedOperationException(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException {} - - @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {} - } - public static class CustomVoidNamespaceSerializer extends TypeSerializer { private static final long serialVersionUID = 1L; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java index 54cac1ea17746..a9bd1f694a30a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java @@ -149,14 +149,6 @@ public abstract class StateBackendTestBase exten @Rule public final ExpectedException expectedException = ExpectedException.none(); - /** - * The serialization timeliness behaviour of the state backend under test. - */ - public enum BackendSerializationTimeliness { - ON_ACCESS, - ON_CHECKPOINTS - } - // lazily initialized stream storage private CheckpointStorageLocation checkpointStorageLocation; @@ -3145,6 +3137,57 @@ public void testMapStateDefaultValue() throws Exception { backend.dispose(); } + @Test + public void testSnapshotNonAccessedState() throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); + SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); + AbstractKeyedStateBackend backend = createKeyedBackend(StringSerializer.INSTANCE); + + final String stateName = "test-name"; + try { + MapStateDescriptor kvId = new MapStateDescriptor<>(stateName, Integer.class, String.class); + MapState mapState = backend + .getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId); + + // write some state to be snapshotted + backend.setCurrentKey("1"); + mapState.put(11, "foo"); + backend.setCurrentKey("2"); + mapState.put(8, "bar"); + backend.setCurrentKey("3"); + mapState.put(91, "hello world"); + + // take a snapshot, and then restore backend with snapshot + KeyedStateHandle snapshot = runSnapshot( + backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), + sharedStateRegistry); + backend.dispose(); + + backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot); + + // now take a snapshot again without accessing the state + snapshot = runSnapshot( + backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), + sharedStateRegistry); + backend.dispose(); + + // we restore again and try to access previous state + backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot); + mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId); + + backend.setCurrentKey("1"); + assertEquals("foo", mapState.get(11)); + backend.setCurrentKey("2"); + assertEquals("bar", mapState.get(8)); + backend.setCurrentKey("3"); + assertEquals("hello world", mapState.get(91)); + + snapshot.discardState(); + } finally { + backend.dispose(); + } + } + /** * This test verifies that state is correctly assigned to key groups and that restore * restores the relevant key groups in the backend. diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java new file mode 100644 index 0000000000000..de1f2bd962cd2 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.runtime.testutils.statemigration.TestType; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test suit for {@link StateSerializerProvider}. + */ +public class StateSerializerProviderTest { + + // -------------------------------------------------------------------------------- + // Tests for #currentSchemaSerializer() + // -------------------------------------------------------------------------------- + + @Test + public void testCurrentSchemaSerializerForNewStateSerializerProvider() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testCurrentSchemaSerializerForRestoredStateSerializerProvider() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + // -------------------------------------------------------------------------------- + // Tests for #previousSchemaSerializer() + // -------------------------------------------------------------------------------- + + @Test(expected = UnsupportedOperationException.class) + public void testPreviousSchemaSerializerForNewStateSerializerProvider() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + + // this should fail with an exception + testProvider.previousSchemaSerializer(); + } + + @Test + public void testPreviousSchemaSerializerForRestoredStateSerializerProvider() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testLazyInstantiationOfPreviousSchemaSerializer() { + // create the provider with an exception throwing snapshot; + // this would throw an exception if the restore serializer was eagerly accessed + StateSerializerProvider testProvider = + StateSerializerProvider.fromRestoredState(new ExceptionThrowingSerializerSnapshot()); + + try { + // if we fail here, that means the restore serializer was indeed lazily accessed + testProvider.previousSchemaSerializer(); + fail("expected to fail when accessing the restore serializer."); + } catch (Exception expected) { + // success + } + } + + // -------------------------------------------------------------------------------- + // Tests for #registerNewSerializerForRestoredState(TypeSerializer) + // -------------------------------------------------------------------------------- + + @Test(expected = UnsupportedOperationException.class) + public void testRegisterNewSerializerWithNewStateSerializerProviderShouldFail() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); + } + + @Test(expected = UnsupportedOperationException.class) + public void testRegisterNewSerializerTwiceWithNewStateSerializerProviderShouldFail() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + + testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); + + // second registration should fail + testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); + } + + @Test + public void testRegisterNewCompatibleAsIsSerializer() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + + // register compatible serializer for state + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.registerNewSerializerForRestoredState(new TestType.V1TestTypeSerializer()); + assertTrue(schemaCompatibility.isCompatibleAsIs()); + + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testRegisterNewCompatibleAfterMigrationSerializer() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + + // register serializer that requires migration for state + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); + assertTrue(schemaCompatibility.isCompatibleAfterMigration()); + } + + @Test + public void testRegisterIncompatibleSerializer() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + + // register serializer that requires migration for state + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.registerNewSerializerForRestoredState(new TestType.IncompatibleTestTypeSerializer()); + assertTrue(schemaCompatibility.isIncompatible()); + + try { + // a serializer for the current schema will no longer be accessible + testProvider.currentSchemaSerializer(); + } catch (Exception excepted) { + // success + } + } + + // -------------------------------------------------------------------------------- + // Utilities + // -------------------------------------------------------------------------------- + + public static class ExceptionThrowingSerializerSnapshot implements TypeSerializerSnapshot { + + @Override + public TypeSerializer restoreSerializer() { + throw new UnsupportedOperationException(); + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { + throw new UnsupportedOperationException(); + } + + @Override + public int getCurrentVersion() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java index 54cd9c9c83730..ba79004bc7938 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java @@ -19,15 +19,11 @@ package org.apache.flink.runtime.state.heap; import org.apache.flink.api.common.state.StateDescriptor; -import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.runtime.state.ArrayListSerializer; import org.apache.flink.runtime.state.KeyGroupRange; @@ -564,103 +560,4 @@ public TypeSerializer getKeySerializer() { } } - /** - * Serializer that can be disabled. Duplicates are still enabled, so we can check that - * serializers are duplicated. - */ - static class TestDuplicateSerializer extends TypeSerializer { - - private static final long serialVersionUID = 1L; - - private static final Integer ZERO = 0; - - private boolean disabled; - - public TestDuplicateSerializer() { - this.disabled = false; - } - - @Override - public boolean isImmutableType() { - return true; - } - - @Override - public TypeSerializer duplicate() { - return new TestDuplicateSerializer(); - } - - @Override - public Integer createInstance() { - return ZERO; - } - - @Override - public Integer copy(Integer from) { - return from; - } - - @Override - public Integer copy(Integer from, Integer reuse) { - return from; - } - - @Override - public int getLength() { - return 4; - } - - @Override - public void serialize(Integer record, DataOutputView target) throws IOException { - Assert.assertFalse(disabled); - target.writeInt(record); - } - - @Override - public Integer deserialize(DataInputView source) throws IOException { - Assert.assertFalse(disabled); - return source.readInt(); - } - - @Override - public Integer deserialize(Integer reuse, DataInputView source) throws IOException { - Assert.assertFalse(disabled); - return deserialize(source); - } - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - Assert.assertFalse(disabled); - target.writeInt(source.readInt()); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof TestDuplicateSerializer; - } - - @Override - public boolean canEqual(Object obj) { - return obj instanceof TestDuplicateSerializer; - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } - - public void disable() { - this.disabled = true; - } - - @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { - throw new UnsupportedOperationException(); - } - - @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - throw new UnsupportedOperationException(); - } - } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/TestDuplicateSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/TestDuplicateSerializer.java new file mode 100644 index 0000000000000..83e249be8f805 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/TestDuplicateSerializer.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.heap; + +import org.apache.flink.api.common.typeutils.CompatibilityResult; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import org.junit.Assert; + +import java.io.IOException; + +/** + * This serializer can be used to test serializer duplication. The serializer that can be disabled. Duplicates are still + * enabled, so we can check that serializers are duplicated. + */ +public class TestDuplicateSerializer extends TypeSerializer { + + private static final long serialVersionUID = 1L; + + private static final Integer ZERO = 0; + + private boolean disabled; + + public TestDuplicateSerializer() { + this.disabled = false; + } + + @Override + public boolean isImmutableType() { + return true; + } + + @Override + public TypeSerializer duplicate() { + return new TestDuplicateSerializer(); + } + + @Override + public Integer createInstance() { + return ZERO; + } + + @Override + public Integer copy(Integer from) { + return from; + } + + @Override + public Integer copy(Integer from, Integer reuse) { + return from; + } + + @Override + public int getLength() { + return 4; + } + + @Override + public void serialize(Integer record, DataOutputView target) throws IOException { + Assert.assertFalse(disabled); + target.writeInt(record); + } + + @Override + public Integer deserialize(DataInputView source) throws IOException { + Assert.assertFalse(disabled); + return source.readInt(); + } + + @Override + public Integer deserialize(Integer reuse, DataInputView source) throws IOException { + Assert.assertFalse(disabled); + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + Assert.assertFalse(disabled); + target.writeInt(source.readInt()); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof TestDuplicateSerializer; + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof TestDuplicateSerializer; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + public void disable() { + this.disabled = true; + } + + @Override + public TypeSerializerConfigSnapshot snapshotConfiguration() { + throw new UnsupportedOperationException(); + } + + @Override + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + throw new UnsupportedOperationException(); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LocalTaskManagerLocation.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LocalTaskManagerLocation.java index 60dddbb8b120c..1c46891e9e253 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LocalTaskManagerLocation.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LocalTaskManagerLocation.java @@ -30,6 +30,6 @@ public class LocalTaskManagerLocation extends TaskManagerLocation { private static final long serialVersionUID = 2396142513336559461L; public LocalTaskManagerLocation() { - super(ResourceID.generate(), InetAddress.getLoopbackAddress(), -1); + super(ResourceID.generate(), InetAddress.getLoopbackAddress(), 42); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java index 14aecc700ad2b..ddae8462539ac 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java @@ -109,7 +109,7 @@ public void testCancelAsyncProducerAndConsumer() throws Exception { // Submit job and wait until running flink.runDetached(jobGraph); - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> flink.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, @@ -160,7 +160,7 @@ public void testCancelAsyncProducerAndConsumer() throws Exception { .get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); // wait until the job is canceled - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> flink.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java new file mode 100644 index 0000000000000..e3b0a066e5158 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.testutils.statemigration; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.runtime.state.Keyed; +import org.apache.flink.runtime.state.PriorityComparable; +import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement; +import org.junit.Assert; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Objects; + +/** + * A data type used as state in state migration tests. + * + *

    This is implemented so that the type can also be used as keyed priority queue state. + */ +public class TestType implements HeapPriorityQueueElement, PriorityComparable, Keyed { + + private int index; + + private final int value; + private final String key; + + public TestType(String key, int value) { + this.key = key; + this.value = value; + } + + @Override + public String getKey() { + return key; + } + + public int getValue() { + return value; + } + + @Override + public int comparePriorityTo(@Nonnull TestType other) { + return Integer.compare(value, other.value); + } + + @Override + public int getInternalIndex() { + return index; + } + + @Override + public void setInternalIndex(int newIndex) { + this.index = newIndex; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof TestType)) { + return false; + } + + if (obj == this) { + return true; + } + + TestType other = (TestType) obj; + return Objects.equals(key, other.key) && value == other.value; + } + + @Override + public int hashCode() { + return 31 * key.hashCode() + value; + } + + /** + * A serializer that read / writes {@link TestType} in schema version 1. + */ + public static class V1TestTypeSerializer extends TestTypeSerializerBase { + private static final long serialVersionUID = 5053346160938769779L; + + @Override + public void serialize(TestType record, DataOutputView target) throws IOException { + target.writeUTF(record.getKey()); + target.writeInt(record.getValue()); + } + + @Override + public TestType deserialize(DataInputView source) throws IOException { + return new TestType(source.readUTF(), source.readInt()); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new V1TestTypeSerializerSnapshot(); + } + } + + /** + * A serializer that read / writes {@link TestType} in schema version 2. + * Migration is required if the state was previously written with {@link V1TestTypeSerializer}. + */ + public static class V2TestTypeSerializer extends TestTypeSerializerBase { + + private static final long serialVersionUID = 7199590310936186578L; + + private static final String RANDOM_PAYLOAD = "random-payload"; + + @Override + public void serialize(TestType record, DataOutputView target) throws IOException { + target.writeUTF(record.getKey()); + target.writeUTF(RANDOM_PAYLOAD); + target.writeInt(record.getValue()); + target.writeBoolean(true); + } + + @Override + public TestType deserialize(DataInputView source) throws IOException { + String key = source.readUTF(); + Assert.assertEquals(RANDOM_PAYLOAD, source.readUTF()); + int value = source.readInt(); + Assert.assertTrue(source.readBoolean()); + + return new TestType(key, value); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new V1TestTypeSerializerSnapshot(); + } + } + + /** + * A serializer that is meant to be incompatible with any of the serializers. + */ + public static class IncompatibleTestTypeSerializer extends TestTypeSerializerBase { + + private static final long serialVersionUID = -2959080770523247215L; + + @Override + public void serialize(TestType record, DataOutputView target) throws IOException { + throw new UnsupportedOperationException("This is an incompatible serializer; shouldn't be used."); + } + + @Override + public TestType deserialize(DataInputView source) throws IOException { + throw new UnsupportedOperationException("This is an incompatible serializer; shouldn't be used."); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + throw new UnsupportedOperationException("This is an incompatible serializer; shouldn't be used."); + } + } + + public static abstract class TestTypeSerializerBase extends TypeSerializer { + + private static final long serialVersionUID = 256299937766275871L; + + // -------------------------------------------------------------------------------- + // Miscellaneous serializer methods + // -------------------------------------------------------------------------------- + + @Override + public TestType copy(TestType from) { + return new TestType(from.getKey(), from.getValue()); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + serialize(deserialize(source), target); + } + + @Override + public TestType deserialize(TestType reuse, DataInputView source) throws IOException { + return deserialize(source); + } + + @Override + public TestType copy(TestType from, TestType reuse) { + return copy(from); + } + + @Override + public TestType createInstance() { + throw new UnsupportedOperationException(); + } + + @Override + public TypeSerializer duplicate() { + return this; + } + + @Override + public boolean isImmutableType() { + return false; + } + + @Override + public int getLength() { + return -1; + } + + @Override + public boolean canEqual(Object obj) { + return getClass().equals(obj.getClass()); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public boolean equals(Object obj) { + return obj == this; + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java new file mode 100644 index 0000000000000..b2b802a30c2c5 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.testutils.statemigration; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; + +/** + * Snapshot class for {@link TestType.V1TestTypeSerializer}. + */ +public class V1TestTypeSerializerSnapshot implements TypeSerializerSnapshot { + + @Override + public int getCurrentVersion() { + return 1; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { + if (newSerializer instanceof TestType.V1TestTypeSerializer) { + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } else if (newSerializer instanceof TestType.V2TestTypeSerializer) { + return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); + } else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public TypeSerializer restoreSerializer() { + return new TestType.V1TestTypeSerializer(); + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java new file mode 100644 index 0000000000000..3cd4fff8d8663 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.testutils.statemigration; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; + +/** + * Snapshot class for {@link TestType.V2TestTypeSerializer}. + */ +public class V2TestTypeSerializerSnapshot implements TypeSerializerSnapshot { + + @Override + public int getCurrentVersion() { + return 1; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { + if (newSerializer instanceof TestType.V2TestTypeSerializer) { + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public TypeSerializer restoreSerializer() { + return new TestType.V2TestTypeSerializer(); + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + } +} diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index 033d505032229..cc6ebb02737f5 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -75,15 +75,26 @@ do fi done -log_setting="" +if [ "$FLINK_IDENT_STRING" = "" ]; then + FLINK_IDENT_STRING="$USER" +fi + +MODE=$1 +LOG=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-scala-shell-$MODE-$HOSTNAME.log -if [[ $1 = "yarn" ]] +if [[ ($MODE = "local") || ($MODE = "remote") ]] then -FLINK_CLASSPATH=$FLINK_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR -log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-scala-shell-yarn-$HOSTNAME.log -log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-yarn-session.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback-yarn.xml" + LOG4J_CONFIG=log4j.properties + LOGBACK_CONFIG=logback.xml +elif [[ $1 = "yarn" ]] +then + LOG4J_CONFIG=log4j-yarn-session.properties + LOGBACK_CONFIG=logback-yarn.xml + FLINK_CLASSPATH=$FLINK_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR fi +log_setting="-Dlog.file="$LOG" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/$LOG4J_CONFIG -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/$LOGBACK_CONFIG" + if ${EXTERNAL_LIB_FOUND} then java -Dscala.color -cp "$FLINK_CLASSPATH" $log_setting org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml index e7fc6e13c2b50..bf1ea5f03b3c4 100644 --- a/flink-scala/pom.xml +++ b/flink-scala/pom.xml @@ -244,6 +244,7 @@ under the License. + src/test/java src/test/scala diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java deleted file mode 100644 index 8ad4e09a1e26d..0000000000000 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.scala.typeutils; - -import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializer; - -import scala.util.Either; - -/** - * Configuration snapshot for serializers of Scala's {@link Either} type, - * containing configuration snapshots of the Left and Right serializers. - */ -public class ScalaEitherSerializerConfigSnapshot - extends CompositeTypeSerializerConfigSnapshot> { - - private static final int VERSION = 1; - - /** This empty nullary constructor is required for deserializing the configuration. */ - public ScalaEitherSerializerConfigSnapshot() {} - - public ScalaEitherSerializerConfigSnapshot(TypeSerializer leftSerializer, TypeSerializer rightSerializer) { - super(leftSerializer, rightSerializer); - } - - @Override - public int getVersion() { - return VERSION; - } -} diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java new file mode 100644 index 0000000000000..b67e47b4effa2 --- /dev/null +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.scala.typeutils; + +import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +import scala.util.Either; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * Configuration snapshot for serializers of Scala's {@link Either} type, + * containing configuration snapshots of the Left and Right serializers. + */ +public class ScalaEitherSerializerSnapshot implements TypeSerializerSnapshot> { + + private static final int CURRENT_VERSION = 1; + + private CompositeSerializerSnapshot nestedLeftRightSerializerSnapshot; + + /** + * Constructor for read instantiation. + */ + public ScalaEitherSerializerSnapshot() {} + + /** + * Constructor to create the snapshot for writing. + */ + public ScalaEitherSerializerSnapshot(TypeSerializer leftSerializer, TypeSerializer rightSerializer) { + Preconditions.checkNotNull(leftSerializer); + Preconditions.checkNotNull(rightSerializer); + this.nestedLeftRightSerializerSnapshot = new CompositeSerializerSnapshot(leftSerializer, rightSerializer); + } + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public TypeSerializer> restoreSerializer() { + return new EitherSerializer<>( + nestedLeftRightSerializerSnapshot.getRestoreSerializer(0), + nestedLeftRightSerializerSnapshot.getRestoreSerializer(1)); + } + + @Override + public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility( + TypeSerializer> newSerializer) { + checkState(nestedLeftRightSerializerSnapshot != null); + + if (newSerializer instanceof EitherSerializer) { + EitherSerializer serializer = (EitherSerializer) newSerializer; + + return nestedLeftRightSerializerSnapshot.resolveCompatibilityWithNested( + TypeSerializerSchemaCompatibility.compatibleAsIs(), + serializer.getLeftSerializer(), + serializer.getRightSerializer()); + } + else { + return TypeSerializerSchemaCompatibility.incompatible(); + } + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + nestedLeftRightSerializerSnapshot.writeCompositeSnapshot(out); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.nestedLeftRightSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + } +} diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala index 14f2196b9ccd3..68432a6f1a502 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala @@ -116,21 +116,22 @@ class EitherSerializer[A, B]( 31 * leftSerializer.hashCode() + rightSerializer.hashCode() } + def getLeftSerializer: TypeSerializer[A] = leftSerializer + + def getRightSerializer: TypeSerializer[B] = rightSerializer + // -------------------------------------------------------------------------------------------- // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - override def snapshotConfiguration(): ScalaEitherSerializerConfigSnapshot[A, B] = { - new ScalaEitherSerializerConfigSnapshot[A, B](leftSerializer, rightSerializer) + override def snapshotConfiguration(): ScalaEitherSerializerSnapshot[A, B] = { + new ScalaEitherSerializerSnapshot[A, B](leftSerializer, rightSerializer) } override def ensureCompatibility( configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Either[A, B]] = { configSnapshot match { - case eitherSerializerConfig: ScalaEitherSerializerConfigSnapshot[A, B] => - checkCompatibility(eitherSerializerConfig) - // backwards compatibility path; // Flink versions older or equal to 1.5.x uses a // EitherSerializerConfigSnapshot as the snapshot diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala index eff57b68755b7..d779afa9d1a23 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala @@ -18,7 +18,7 @@ package org.apache.flink.api.scala.typeutils import org.apache.flink.annotation.Internal -import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot} +import org.apache.flink.api.common.typeutils._ import org.apache.flink.core.memory.{DataInputView, DataOutputView} /** @@ -56,8 +56,8 @@ class NothingSerializer extends TypeSerializer[Any] { override def deserialize(reuse: Any, source: DataInputView): Any = throw new RuntimeException("This must not be used. You encountered a bug.") - override def snapshotConfiguration(): TypeSerializerConfigSnapshot[Any] = - throw new RuntimeException("This must not be used. You encountered a bug.") + override def snapshotConfiguration(): TypeSerializerSnapshot[Any] = + new NothingSerializerSnapshot override def ensureCompatibility( configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Any] = @@ -78,3 +78,7 @@ class NothingSerializer extends TypeSerializer[Any] { classOf[NothingSerializer].hashCode() } } + +class NothingSerializerSnapshot + extends SimpleTypeSerializerSnapshot[Any](classOf[NothingSerializer]) { +} diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala index b5d069fbe9396..71a482aba1db4 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala @@ -40,7 +40,7 @@ abstract class TraversableSerializer[T <: TraversableOnce[E], E]( override def duplicate = { val duplicateElementSerializer = elementSerializer.duplicate() - if (duplicateElementSerializer == elementSerializer) { + if (duplicateElementSerializer eq elementSerializer) { // is not stateful, so return ourselves this } else { diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala index 786c12839381c..03324568bb42f 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala @@ -32,12 +32,21 @@ import scala.util.{Failure, Success, Try} @SerialVersionUID(-3052182891252564491L) class TrySerializer[A]( private val elemSerializer: TypeSerializer[A], - private val executionConfig: ExecutionConfig) + private val throwableSerializer: TypeSerializer[Throwable]) extends TypeSerializer[Try[A]] { - override def duplicate: TrySerializer[A] = this + private[typeutils] def this(elemSerializer: TypeSerializer[A], + executionConfig: ExecutionConfig) = { + this( + elemSerializer, + new KryoSerializer[Throwable](classOf[Throwable], executionConfig) + ) + } - val throwableSerializer = new KryoSerializer[Throwable](classOf[Throwable], executionConfig) + override def duplicate: TrySerializer[A] = new TrySerializer[A]( + elemSerializer.duplicate(), + throwableSerializer.duplicate() + ) override def createInstance: Try[A] = { Failure(new RuntimeException("Empty Failure")) @@ -97,7 +106,7 @@ class TrySerializer[A]( } override def hashCode(): Int = { - 31 * elemSerializer.hashCode() + executionConfig.hashCode() + 31 * elemSerializer.hashCode() + throwableSerializer.hashCode() } // -------------------------------------------------------------------------------------------- diff --git a/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshotMigrationTest.java b/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshotMigrationTest.java new file mode 100644 index 0000000000000..9cd8b5d55b5dd --- /dev/null +++ b/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshotMigrationTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.scala.typeutils; + +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; + +import scala.util.Either; + +/** + * Migration test for the {@link ScalaEitherSerializerSnapshot}. + */ +public class ScalaEitherSerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase> { + + private static final String DATA = "flink-1.6-scala-either-serializer-data"; + private static final String SNAPSHOT = "flink-1.6-scala-either-serializer-snapshot"; + + public ScalaEitherSerializerSnapshotMigrationTest() { + super( + TestSpecification.>builder( + "1.6-scala-either-serializer", + EitherSerializer.class, + ScalaEitherSerializerSnapshot.class) + .withSerializerProvider(() -> new EitherSerializer<>(IntSerializer.INSTANCE, StringSerializer.INSTANCE)) + .withSnapshotDataLocation(SNAPSHOT) + .withTestData(DATA, 10) + ); + } + +} diff --git a/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-data b/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-data new file mode 100644 index 0000000000000..203067c5a04a3 Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-data differ diff --git a/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-snapshot b/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-snapshot new file mode 100644 index 0000000000000..123940604969e Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.6-scala-either-serializer-snapshot differ diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/KryoGenericTypeSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/KryoGenericTypeSerializerTest.scala index 1062e654e0839..a0c285fa8bf76 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/KryoGenericTypeSerializerTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/KryoGenericTypeSerializerTest.scala @@ -102,14 +102,14 @@ class KryoGenericTypeSerializerTest { def testScalaListSerialization(): Unit = { val a = List(42,1,49,1337) - runTests(a) + runTests(Seq(a)) } @Test def testScalaMutablelistSerialization(): Unit = { val a = scala.collection.mutable.ListBuffer(42,1,49,1337) - runTests(a) + runTests(Seq(a)) } @Test @@ -133,7 +133,7 @@ class KryoGenericTypeSerializerTest { val c = ComplexType("1337", 1, List(1)) val list = List(a, b, c) - runTests(list) + runTests(Seq(list)) } @Test @@ -143,7 +143,7 @@ class KryoGenericTypeSerializerTest { val c = new DerivedType2("bar", "foo") val list = List(a,b,c) - runTests(list) + runTests(Seq(list)) } diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/ScalaSpecialTypesSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/ScalaSpecialTypesSerializerTest.scala index 555359f89abf6..8809e777acb49 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/ScalaSpecialTypesSerializerTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/ScalaSpecialTypesSerializerTest.scala @@ -17,11 +17,16 @@ */ package org.apache.flink.api.scala.runtime +import java.lang.{Boolean => JBoolean} +import java.util.function.BiFunction + import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.common.typeutils.{SerializerTestInstance, TypeSerializer} import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer import org.apache.flink.api.scala._ +import org.apache.flink.testutils.DeeplyEqualsChecker +import org.apache.flink.testutils.DeeplyEqualsChecker.CustomEqualityChecker import org.junit.Assert._ import org.junit.{Assert, Test} @@ -143,12 +148,67 @@ class ScalaSpecialTypesSerializerTest { } } +object ScalaSpecialTypesSerializerTestInstance { + + val isTraversable: BiFunction[AnyRef, AnyRef, JBoolean] = + new BiFunction[AnyRef, AnyRef, JBoolean] { + override def apply(o1: scala.AnyRef, o2: scala.AnyRef): JBoolean = + o1.isInstanceOf[TraversableOnce[_]] && o2.isInstanceOf[TraversableOnce[_]] + } + + val isFailure: BiFunction[AnyRef, AnyRef, JBoolean] = + new BiFunction[AnyRef, AnyRef, JBoolean] { + override def apply(o1: scala.AnyRef, o2: scala.AnyRef): JBoolean = + o1.isInstanceOf[Failure[_]] && o2.isInstanceOf[Failure[_]] + } + + val compareTraversable: CustomEqualityChecker = + new CustomEqualityChecker { + override def check( + o1: AnyRef, + o2: AnyRef, + checker: DeeplyEqualsChecker): Boolean = { + val s1 = o1.asInstanceOf[TraversableOnce[_]].toIterator + val s2 = o2.asInstanceOf[TraversableOnce[_]].toIterator + + while (s1.hasNext && s2.hasNext) { + val l = s1.next + val r = s2.next + if (!checker.deepEquals(l, r)) { + return false + } + } + !s1.hasNext && !s2.hasNext + } + } + + val compareFailure: CustomEqualityChecker = + new CustomEqualityChecker { + override def check( + o1: AnyRef, + o2: AnyRef, + checker: DeeplyEqualsChecker): Boolean = { + o1.asInstanceOf[Failure[_]].exception.getMessage + .equals(o2.asInstanceOf[Failure[_]].exception.getMessage) + } + } +} + class ScalaSpecialTypesSerializerTestInstance[T]( serializer: TypeSerializer[T], typeClass: Class[T], length: Int, testData: Array[T]) - extends SerializerTestInstance[T](serializer, typeClass, length, testData: _*) { + extends SerializerTestInstance[T]( + new DeeplyEqualsChecker() + .withCustomCheck(ScalaSpecialTypesSerializerTestInstance.isTraversable, + ScalaSpecialTypesSerializerTestInstance.compareTraversable) + .withCustomCheck(ScalaSpecialTypesSerializerTestInstance.isFailure, + ScalaSpecialTypesSerializerTestInstance.compareFailure), + serializer, + typeClass, + length, + testData: _*) { @Test override def testInstantiate(): Unit = { @@ -173,27 +233,6 @@ class ScalaSpecialTypesSerializerTestInstance[T]( } } } - - override protected def deepEquals(message: String, should: T, is: T) { - should match { - case trav: TraversableOnce[_] => - val isTrav = is.asInstanceOf[TraversableOnce[_]] - assertEquals(message, trav.size, isTrav.size) - val it = trav.toIterator - val isIt = isTrav.toIterator - while (it.hasNext) { - val should = it.next() - val is = isIt.next() - assertEquals(message, should, is) - } - - case Failure(t) => - is.asInstanceOf[Failure[_]].exception.equals(t) - - case _ => - super.deepEquals(message, should, is) - } - } } object WeekDay extends Enumeration { diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala index e177e7c4ac42f..3df28ed5816d6 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala @@ -177,22 +177,5 @@ class TraversableSerializerTestInstance[T]( } } - override protected def deepEquals(message: String, should: T, is: T) { - should match { - case trav: TraversableOnce[_] => - val isTrav = is.asInstanceOf[TraversableOnce[_]] - assertEquals(message, trav.size, isTrav.size) - val it = trav.toIterator - val isIt = isTrav.toIterator - while (it.hasNext) { - val should = it.next() - val is = isIt.next() - assertEquals(message, should, is) - } - - case _ => - super.deepEquals(message, should, is) - } - } } diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala index 7a425900c96c9..3bdc54aec9118 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala @@ -17,19 +17,57 @@ */ package org.apache.flink.api.scala.runtime +import java.lang.{Boolean => JBoolean} +import java.util.function.BiFunction + +import org.apache.flink.api.common.typeutils.{SerializerTestInstance, TypeSerializer} +import org.apache.flink.testutils.DeeplyEqualsChecker +import org.apache.flink.testutils.DeeplyEqualsChecker.CustomEqualityChecker import org.junit.Assert._ -import org.apache.flink.api.common.typeutils.SerializerTestInstance -import org.apache.flink.api.common.typeutils.TypeSerializer -import org.junit.Assert import org.junit.Test +object TupleSerializerTestInstance { + val isProduct: BiFunction[AnyRef, AnyRef, JBoolean] = + new BiFunction[AnyRef, AnyRef, JBoolean] { + override def apply(o1: scala.AnyRef, o2: scala.AnyRef): JBoolean = + o1.isInstanceOf[Product] && o2.isInstanceOf[Product] + } + + val compareProduct: CustomEqualityChecker = + new CustomEqualityChecker { + override def check( + o1: AnyRef, + o2: AnyRef, + checker: DeeplyEqualsChecker): Boolean = { + val p1 = o1.asInstanceOf[Product].productIterator + val p2 = o2.asInstanceOf[Product].productIterator + + while (p1.hasNext && p2.hasNext) { + val l = p1.next + val r = p2.next + if (!checker.deepEquals(l, r)) { + return false + } + } + !p1.hasNext && !p2.hasNext + } + } +} + class TupleSerializerTestInstance[T <: Product] ( serializer: TypeSerializer[T], typeClass: Class[T], length: Int, testData: Array[T]) - extends SerializerTestInstance[T](serializer, typeClass, length, testData: _*) { + extends SerializerTestInstance[T]( + new DeeplyEqualsChecker() + .withCustomCheck(TupleSerializerTestInstance.isProduct, + TupleSerializerTestInstance.compareProduct), + serializer, + typeClass, + length, + testData: _*) { @Test override def testInstantiate(): Unit = { @@ -51,40 +89,5 @@ class TupleSerializerTestInstance[T <: Product] ( } } } - - protected override def deepEquals(message: String, shouldTuple: T, isTuple: T) { - Assert.assertEquals(shouldTuple.productArity, isTuple.productArity) - for (i <- 0 until shouldTuple.productArity) { - val should = shouldTuple.productElement(i) - val is = isTuple.productElement(i) - if (should.getClass.isArray) { - should match { - case booleans: Array[Boolean] => - Assert.assertTrue(message, booleans.sameElements(is.asInstanceOf[Array[Boolean]])) - case bytes: Array[Byte] => - assertArrayEquals(message, bytes, is.asInstanceOf[Array[Byte]]) - case shorts: Array[Short] => - assertArrayEquals(message, shorts, is.asInstanceOf[Array[Short]]) - case ints: Array[Int] => - assertArrayEquals(message, ints, is.asInstanceOf[Array[Int]]) - case longs: Array[Long] => - assertArrayEquals(message, longs, is.asInstanceOf[Array[Long]]) - case floats: Array[Float] => - assertArrayEquals(message, floats, is.asInstanceOf[Array[Float]], 0.0f) - case doubles: Array[Double] => - assertArrayEquals(message, doubles, is.asInstanceOf[Array[Double]], 0.0) - case chars: Array[Char] => - assertArrayEquals(message, chars, is.asInstanceOf[Array[Char]]) - case _ => - assertArrayEquals( - message, - should.asInstanceOf[Array[AnyRef]], - is.asInstanceOf[Array[AnyRef]]) - } - } else { - assertEquals(message, should, is) - } - } - } } diff --git a/flink-shaded-curator/src/main/resources/META-INF/NOTICE b/flink-shaded-curator/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..e6b455563d080 --- /dev/null +++ b/flink-shaded-curator/src/main/resources/META-INF/NOTICE @@ -0,0 +1,12 @@ +flink-shaded-curator +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:16.0.1 +- org.apache.curator:curator-client:2.12.0 +- org.apache.curator:curator-framework:2.12.0 +- org.apache.curator:curator-recipes:2.12.0 diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml index 4fa07e23f33e6..5ac239288d910 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml @@ -113,9 +113,9 @@ under the License. - - - + + Apache Flink + diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..f6d007d94d15f --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE @@ -0,0 +1,52 @@ +flink-shaded-hadoop2-uber +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.jamesmurty.utils:java-xmlbuilder:0.4 +- commons-beanutils:commons-beanutils:1.9.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-daemon:commons-daemon:1.0.13 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- org.apache.avro:avro:1.8.2 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- org.apache.zookeeper:zookeeper:3.4.10 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.jcraft:jsch:0.1.42 +- com.thoughtworks.paranamer:paranamer:2.7 +- xmlenc:xmlenc:0.52 + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.0. +See bundled license files for details. + +- javax.activation:activation:1.1 (http://java.sun.com/products/javabeans/jaf/index.jsp) +- javax.servlet:servlet-api:2.5 (https://repo1.maven.org/maven2/javax/servlet/servlet-api/2.5/) +- javax.xml.stream:stax-api:1.0-2 (http://central.maven.org/maven2/javax/xml/stream/stax-api/1.0-2/) + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.1. +See bundled license files for details. + +- com.sun.jersey:jersey-client:1.9 (http://central.maven.org/maven2/com/sun/jersey/jersey-client/1.9/) +- javax.xml.bind:jaxb-api:2.2.2 (https://jaxb.dev.java.net/) + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.0 b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.0 new file mode 100644 index 0000000000000..468e7516a87ff --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.0 @@ -0,0 +1,129 @@ +1. Definitions. + +1.1. "Contributor" means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. "Contributor Version" means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. "Covered Software" means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. "Executable" means the Covered Software in any form other than Source Code. + +1.5. "Initial Developer" means the individual or entity that first makes Original Software available under this License. + +1.6. "Larger Work" means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. "License" means this document. + +1.8. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. "Modifications" means the Source Code and Executable form of any of the following: + +A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + +B. Any new file that contains any part of the Original Software or previous Modification; or + +C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. "Original Software" means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. "Patent Claims" means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. "Source Code" means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. "You" (or "Your") means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + +(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + +(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. + +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. + +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. + +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. + +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. + +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. + +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. + +Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. + +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. + +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as "Participant") alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" (as that term is defined at 48 C.F.R. ¤ 252.227-7014(a)(1)) and "commercial computer software documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.1 b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.1 new file mode 100644 index 0000000000000..75372e3d2c9a4 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.cddlv1.1 @@ -0,0 +1,348 @@ +1. Definitions. + + 1.1. "Contributor" means each individual or entity that creates or + contributes to the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Software, prior Modifications used by a Contributor (if any), and + the Modifications made by that particular Contributor. + + 1.3. "Covered Software" means (a) the Original Software, or (b) + Modifications, or (c) the combination of files containing Original + Software with files containing Modifications, in each case including + portions thereof. + + 1.4. "Executable" means the Covered Software in any form other than + Source Code. + + 1.5. "Initial Developer" means the individual or entity that first + makes Original Software available under this License. + + 1.6. "Larger Work" means a work which combines Covered Software or + portions thereof with code not governed by the terms of this License. + + 1.7. "License" means this document. + + 1.8. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means the Source Code and Executable form of + any of the following: + + A. Any file that results from an addition to, deletion from or + modification of the contents of a file containing Original Software + or previous Modifications; + + B. Any new file that contains any part of the Original Software or + previous Modification; or + + C. Any new file that is contributed or otherwise made available + under the terms of this License. + + 1.10. "Original Software" means the Source Code and Executable form + of computer software code that is originally released under this + License. + + 1.11. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.12. "Source Code" means (a) the common form of computer software + code in which modifications are made and (b) associated + documentation included in or with such code. + + 1.13. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, + this License. For legal entities, "You" includes any entity which + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, the Initial Developer + hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer, to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Software (or portions thereof), with or without Modifications, + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of + Original Software, to make, have made, use, practice, sell, and + offer for sale, and/or otherwise dispose of the Original Software + (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on + the date Initial Developer first distributes or otherwise makes the + Original Software available to a third party under the terms of this + License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: (1) for code that You delete from the Original Software, or + (2) for infringements caused by: (i) the modification of the + Original Software, or (ii) the combination of the Original Software + with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, each Contributor hereby + grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof), either on an + unmodified basis, with other Modifications, as Covered Software + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling + of Modifications made by that Contributor either alone and/or in + combination with its Contributor Version (or portions of such + combination), to make, use, sell, offer for sale, have made, and/or + otherwise dispose of: (1) Modifications made by that Contributor (or + portions thereof); and (2) the combination of Modifications made by + that Contributor with its Contributor Version (or portions of such + combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective + on the date Contributor first distributes or otherwise makes the + Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: (1) for any code that Contributor has deleted from the + Contributor Version; (2) for infringements caused by: (i) third + party modifications of Contributor Version, or (ii) the combination + of Modifications made by that Contributor with other software + (except as part of the Contributor Version) or other devices; or (3) + under Patent Claims infringed by Covered Software in the absence of + Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available + in Executable form must also be made available in Source Code form + and that Source Code form must be distributed only under the terms + of this License. You must include a copy of this License with every + copy of the Source Code form of the Covered Software You distribute + or otherwise make available. You must inform recipients of any such + Covered Software in Executable form as to how they can obtain such + Covered Software in Source Code form in a reasonable manner on or + through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are + governed by the terms of this License. You represent that You + believe Your Modifications are Your original creation(s) and/or You + have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that + identifies You as the Contributor of the Modification. You may not + remove or alter any copyright, patent or trademark notices contained + within the Covered Software, or any notices of licensing or any + descriptive text giving attribution to any Contributor or the + Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in + Source Code form that alters or restricts the applicable version of + this License or the recipients' rights hereunder. You may choose to + offer, and to charge a fee for, warranty, support, indemnity or + liability obligations to one or more recipients of Covered Software. + However, you may do so only on Your own behalf, and not on behalf of + the Initial Developer or any Contributor. You must make it + absolutely clear that any such warranty, support, indemnity or + liability obligation is offered by You alone, and You hereby agree + to indemnify the Initial Developer and every Contributor for any + liability incurred by the Initial Developer or such Contributor as a + result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under + the terms of this License or under the terms of a license of Your + choice, which may contain terms different from this License, + provided that You are in compliance with the terms of this License + and that the license for the Executable form does not attempt to + limit or alter the recipient's rights in the Source Code form from + the rights set forth in this License. If You distribute the Covered + Software in Executable form under a different license, You must make + it absolutely clear that any terms which differ from this License + are offered by You alone, not by the Initial Developer or + Contributor. You hereby agree to indemnify the Initial Developer and + every Contributor for any liability incurred by the Initial + Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with + other code not governed by the terms of this License and distribute + the Larger Work as a single product. In such a case, You must make + sure the requirements of this License are fulfilled for the Covered + Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or + new versions of this License from time to time. Each version will be + given a distinguishing version number. Except as provided in Section + 4.3, no one other than the license steward has the right to modify + this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the + Covered Software available under the terms of the version of the + License under which You originally received the Covered Software. If + the Initial Developer includes a notice in the Original Software + prohibiting it from being distributed or otherwise made available + under any subsequent version of the License, You must distribute and + make the Covered Software available under the terms of the version + of the License under which You originally received the Covered + Software. Otherwise, You may also choose to use, distribute or + otherwise make the Covered Software available under the terms of any + subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new + license for Your Original Software, You may create and use a + modified version of this License if You: (a) rename the license and + remove any references to the name of the license steward (except to + note that the license differs from this License); and (b) otherwise + make it clear that the license contains terms which differ from this + License. + +5. DISCLAIMER OF WARRANTY. + + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, + INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE + IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR + NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF + THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE + DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY + OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, + REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN + ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS + AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to + cure such breach within 30 days of becoming aware of the breach. + Provisions which, by their nature, must remain in effect beyond the + termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding + declaratory judgment actions) against Initial Developer or a + Contributor (the Initial Developer or Contributor against whom You + assert such claim is referred to as "Participant") alleging that the + Participant Software (meaning the Contributor Version where the + Participant is a Contributor or the Original Software where the + Participant is the Initial Developer) directly or indirectly + infringes any patent, then any and all rights granted directly or + indirectly to You by such Participant, the Initial Developer (if the + Initial Developer is not the Participant) and all Contributors under + Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice + from Participant terminate prospectively and automatically at the + expiration of such 60 day notice period, unless if within such 60 + day period You withdraw Your claim with respect to the Participant + Software against such Participant either unilaterally or pursuant to + a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant + alleging that the Participant Software directly or indirectly + infringes any patent where such claim is resolved (such as by + license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, + all end user licenses that have been validly granted by You or any + distributor hereunder prior to termination (excluding licenses + granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE + INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF + COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE + TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR + CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT + LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER + FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR + LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE + POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT + APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH + PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH + LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR + LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION + AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + + The Covered Software is a "commercial item," as that term is defined + in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" (as that term is defined at 48 C.F.R. § + 252.227-7014(a)(1)) and "commercial computer software documentation" + as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent + with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 + (June 1995), all U.S. Government End Users acquire Covered Software + with only those rights set forth herein. This U.S. Government Rights + clause is in lieu of, and supersedes, any other FAR, DFAR, or other + clause or provision that addresses Government rights in computer + software under this License. + +9. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + the law of the jurisdiction specified in a notice contained within + the Original Software (except to the extent applicable law, if any, + provides otherwise), excluding such jurisdiction's conflict-of-law + provisions. Any litigation relating to this License shall be subject + to the jurisdiction of the courts located in the jurisdiction and + venue specified in a notice contained within the Original Software, + with the losing party responsible for costs, including, without + limitation, court costs and reasonable attorneys' fees and expenses. + The application of the United Nations Convention on Contracts for + the International Sale of Goods is expressly excluded. Any law or + regulation which provides that the language of a contract shall be + construed against the drafter shall not apply to this License. You + agree that You alone are responsible for compliance with the United + States export administration regulations (and the export control + laws and regulation of any other countries) when You use, distribute + or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.jsch b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.jsch new file mode 100644 index 0000000000000..edd491dfbfbf5 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.jsch @@ -0,0 +1,30 @@ +JSch 0.0.* was released under the GNU LGPL license. Later, we have switched +over to a BSD-style license. + +------------------------------------------------------------------------------ +Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.paranamer b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.paranamer new file mode 100644 index 0000000000000..522176fbca03e --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.paranamer @@ -0,0 +1,28 @@ +[ ParaNamer used to be 'Pubic Domain', but since it includes a small piece of ASM it is now the same license as that: BSD ] + + Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.xmlenc b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.xmlenc new file mode 100644 index 0000000000000..3a70c9bfcdadd --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/licenses/LICENSE.xmlenc @@ -0,0 +1,27 @@ +Copyright 2003-2005, Ernst de Haan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/NOTICE b/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..15680c69694af --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/NOTICE @@ -0,0 +1,31 @@ +flink-shaded-hadoop2 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:11.0.2 +- net.java.dev.jets3t:jets3t:0.9.0 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- commons-httpclient:commons-httpclient:3.1 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +The version of the actual Hadoop dependencies depend on the built. + +- org.apache.hadoop:hadoop-common +- org.apache.hadoop:hadoop-hdfs +- org.apache.hadoop:hadoop-mapreduce-client-core +- org.apache.hadoop:hadoop-yarn-client +- org.apache.hadoop:hadoop-yarn-common +- org.apache.hadoop:hadoop-annotations +- org.apache.hadoop:hadoop-auth +- org.apache.hadoop:hadoop-yarn-api + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.google.protobuf.protobuf-java:2.5.0 diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/licenses/LICENSE.protobuf b/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/licenses/LICENSE.protobuf new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2/src/main/resources/META-INF/licenses/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/NOTICE b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..88c4e4c67a5de --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/NOTICE @@ -0,0 +1,47 @@ +flink-shaded-yarn-tests +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:11.0.2 +- io.netty:netty:3.6.2.Final +- net.java.dev.jets3t:jets3t:0.9.0 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- commons-httpclient:commons-httpclient:3.1 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-jaxrs:1.8.3 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 +- org.codehaus.jackson:jackson-xc:1.8.3 + +The version of the actual Hadoop dependencies depend on the built. + +- org.apache.hadoop:hadoop-annotations +- org.apache.hadoop:hadoop-auth +- org.apache.hadoop:hadoop-common +- org.apache.hadoop:hadoop-hdfs +- org.apache.hadoop:hadoop-mapreduce-client-app +- org.apache.hadoop:hadoop-mapreduce-client-common +- org.apache.hadoop:hadoop-mapreduce-client-core +- org.apache.hadoop:hadoop-mapreduce-client-hs +- org.apache.hadoop:hadoop-mapreduce-client-jobclient +- org.apache.hadoop:hadoop-mapreduce-client-shuffle +- org.apache.hadoop:hadoop-minicluster +- org.apache.hadoop:hadoop-yarn-api +- org.apache.hadoop:hadoop-yarn-client +- org.apache.hadoop:hadoop-yarn-common +- org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice +- org.apache.hadoop:hadoop-yarn-server-common +- org.apache.hadoop:hadoop-yarn-server-nodemanager +- org.apache.hadoop:hadoop-yarn-server-resourcemanager +- org.apache.hadoop:hadoop-yarn-server-tests +- org.apache.hadoop:hadoop-yarn-server-web-proxy + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- asm:asm:3.1 +- com.google.protobuf.protobuf-java:2.5.0 diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.asm b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.asm new file mode 100644 index 0000000000000..c71bb7bac5d4d --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.asm @@ -0,0 +1,27 @@ +ASM: a very small and fast Java bytecode manipulation framework +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.protobuf b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.protobuf new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/src/main/resources/META-INF/licenses/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml index 34b8fb9409729..b7027ae533658 100644 --- a/flink-shaded-hadoop/pom.xml +++ b/flink-shaded-hadoop/pom.xml @@ -105,14 +105,20 @@ under the License. META-INF/*.RSA + + commons-httpclient:commons-httpclient + + META-INF/LICENSE.txt + + - - - + + Apache Flink + diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 50caa0d912a55..a37f8aa8df827 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -29,7 +29,6 @@ import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigConstants; @@ -112,7 +111,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -213,14 +211,6 @@ IS createState( */ private final LinkedHashMap> kvStateInformation; - /** - * Map of state names to their corresponding restored state meta info. - * - *

    TODO this map can be removed when eager-state registration is in place. - * TODO we currently need this cached to check state migration strategies when new serializers are registered. - */ - private final Map restoredKvStateMetaInfos; - /** Number of bytes required to prefix the key groups. */ private final int keyGroupPrefixBytes; @@ -297,7 +287,6 @@ public RocksDBKeyedStateBackend( this.keyGroupPrefixBytes = RocksDBKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(getNumberOfKeyGroups()); this.kvStateInformation = new LinkedHashMap<>(); - this.restoredKvStateMetaInfos = new HashMap<>(); this.writeOptions = new WriteOptions().setDisableWAL(true); @@ -425,7 +414,6 @@ public void dispose() { IOUtils.closeQuietly(dbOptions); IOUtils.closeQuietly(writeOptions); kvStateInformation.clear(); - restoredKvStateMetaInfos.clear(); cleanInstanceBasePath(); } @@ -511,7 +499,6 @@ public void restore(Collection restoreState) throws Exception // clear all meta data kvStateInformation.clear(); - restoredKvStateMetaInfos.clear(); try { RocksDBIncrementalRestoreOperation incrementalRestoreOperation = null; @@ -754,11 +741,13 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio nameBytes, rocksDBKeyedStateBackend.columnOptions); - rocksDBKeyedStateBackend.restoredKvStateMetaInfos.put(restoredMetaInfo.getName(), restoredMetaInfo); - ColumnFamilyHandle columnFamily = rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor); - registeredColumn = new Tuple2<>(columnFamily, null); + // create a meta info for the state on restore; + // this allows us to retain the state in future snapshots even if it wasn't accessed + RegisteredStateMetaInfoBase stateMetaInfo = + RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(restoredMetaInfo); + registeredColumn = new Tuple2<>(columnFamily, stateMetaInfo); rocksDBKeyedStateBackend.kvStateInformation.put(restoredMetaInfo.getName(), registeredColumn); } else { @@ -1069,10 +1058,14 @@ private ColumnFamilyHandle getOrRegisterColumnFamilyHandle( stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); if (null == registeredStateMetaInfoEntry) { + // create a meta info for the state on restore; + // this allows us to retain the state in future snapshots even if it wasn't accessed + RegisteredStateMetaInfoBase stateMetaInfo = + RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot); registeredStateMetaInfoEntry = new Tuple2<>( columnFamilyHandle != null ? columnFamilyHandle : stateBackend.db.createColumnFamily(columnFamilyDescriptor), - null); + stateMetaInfo); stateBackend.registerKvStateInformation( stateMetaInfoSnapshot.getName(), @@ -1159,7 +1152,6 @@ private List createAndRegisterColumnFamilyDescriptors( stateBackend.columnOptions); columnFamilyDescriptors.add(columnFamilyDescriptor); - stateBackend.restoredKvStateMetaInfos.put(stateMetaInfoSnapshot.getName(), stateMetaInfoSnapshot); } return columnFamilyDescriptors; } @@ -1201,9 +1193,13 @@ private void restoreLocalStateIntoFullInstance( ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); + // create a meta info for the state on restore; + // this allows us to retain the state in future snapshots even if it wasn't accessed + RegisteredStateMetaInfoBase stateMetaInfo = + RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot); stateBackend.registerKvStateInformation( stateMetaInfoSnapshot.getName(), - new Tuple2<>(columnFamilyHandle, null)); + new Tuple2<>(columnFamilyHandle, stateMetaInfo)); } // use the restore sst files as the base for succeeding checkpoints @@ -1365,78 +1361,69 @@ private Tuple2 namespaceSerializer, @Nullable StateSnapshotTransformer snapshotTransformer) throws Exception { - Tuple2 stateInfo = + Tuple2 oldStateInfo = kvStateInformation.get(stateDesc.getName()); TypeSerializer stateSerializer = stateDesc.getSerializer(); - RegisteredKeyValueStateBackendMetaInfo newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( - stateDesc.getType(), - stateDesc.getName(), - namespaceSerializer, - stateSerializer, - snapshotTransformer); - - if (stateInfo != null) { - newMetaInfo = migrateStateIfNecessary( - newMetaInfo, + + ColumnFamilyHandle newColumnFamily; + RegisteredKeyValueStateBackendMetaInfo newMetaInfo; + if (oldStateInfo != null) { + @SuppressWarnings("unchecked") + RegisteredKeyValueStateBackendMetaInfo castedMetaInfo = (RegisteredKeyValueStateBackendMetaInfo) oldStateInfo.f1; + + newMetaInfo = updateRestoredStateMetaInfo( + Tuple2.of(oldStateInfo.f0, castedMetaInfo), stateDesc, namespaceSerializer, stateSerializer, - stateInfo); + snapshotTransformer); - stateInfo.f1 = newMetaInfo; + oldStateInfo.f1 = newMetaInfo; + newColumnFamily = oldStateInfo.f0; } else { - ColumnFamilyHandle columnFamily = createColumnFamily(stateDesc.getName()); + newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( + stateDesc.getType(), + stateDesc.getName(), + namespaceSerializer, + stateSerializer, + snapshotTransformer); - stateInfo = Tuple2.of(columnFamily, newMetaInfo); - registerKvStateInformation(stateDesc.getName(), stateInfo); + newColumnFamily = createColumnFamily(stateDesc.getName()); + registerKvStateInformation(stateDesc.getName(), Tuple2.of(newColumnFamily, newMetaInfo)); } - return Tuple2.of(stateInfo.f0, newMetaInfo); + return Tuple2.of(newColumnFamily, newMetaInfo); } - private RegisteredKeyValueStateBackendMetaInfo migrateStateIfNecessary( - RegisteredKeyValueStateBackendMetaInfo newMetaInfo, + private RegisteredKeyValueStateBackendMetaInfo updateRestoredStateMetaInfo( + Tuple2> oldStateInfo, StateDescriptor stateDesc, TypeSerializer namespaceSerializer, TypeSerializer stateSerializer, - Tuple2 stateInfo) throws Exception { - - StateMetaInfoSnapshot restoredMetaInfoSnapshot = restoredKvStateMetaInfos.get(stateDesc.getName()); - - Preconditions.checkState( - restoredMetaInfoSnapshot != null, - "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," + - " but its corresponding restored snapshot cannot be found."); + @Nullable StateSnapshotTransformer snapshotTransformer) throws Exception { @SuppressWarnings("unchecked") - TypeSerializerSnapshot namespaceSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot( - StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString())); + RegisteredKeyValueStateBackendMetaInfo restoredKvStateMetaInfo = oldStateInfo.f1; + + restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); - TypeSerializerSchemaCompatibility namespaceCompatibility = - namespaceSerializerSnapshot.resolveSchemaCompatibility(namespaceSerializer); - if (!namespaceCompatibility.isCompatibleAsIs()) { + TypeSerializerSchemaCompatibility s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); + if (!s.isCompatibleAsIs()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } - @SuppressWarnings("unchecked") - TypeSerializerSnapshot stateSerializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot( - StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString())); - - RegisteredKeyValueStateBackendMetaInfo.checkStateMetaInfo(restoredMetaInfoSnapshot, stateDesc); - - TypeSerializerSchemaCompatibility stateCompatibility = - stateSerializerSnapshot.resolveSchemaCompatibility(stateSerializer); + restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); - if (stateCompatibility.isCompatibleAfterMigration()) { - migrateStateValues(stateDesc, stateInfo, restoredMetaInfoSnapshot, newMetaInfo, stateSerializer); - } else if (stateCompatibility.isIncompatible()) { + TypeSerializerSchemaCompatibility newStateSerializerCompatibility = + restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); + if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { + migrateStateValues(stateDesc, oldStateInfo); + } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } - return newMetaInfo; + return restoredKvStateMetaInfo; } /** @@ -1446,10 +1433,7 @@ private RegisteredKeyValueStateBackendMetaInfo m */ private void migrateStateValues( StateDescriptor stateDesc, - Tuple2 stateInfo, - StateMetaInfoSnapshot restoredMetaInfoSnapshot, - RegisteredKeyValueStateBackendMetaInfo newMetaInfo, - TypeSerializer newStateSerializer) throws Exception { + Tuple2> stateMetaInfo) throws Exception { if (stateDesc.getType() == StateDescriptor.Type.MAP) { throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed." + @@ -1471,7 +1455,7 @@ private void migrateStateValues( } State state = stateFactory.createState( stateDesc, - Tuple2.of(stateInfo.f0, newMetaInfo), + stateMetaInfo, RocksDBKeyedStateBackend.this); if (!(state instanceof AbstractRocksDBState)) { throw new FlinkRuntimeException( @@ -1483,16 +1467,11 @@ private void migrateStateValues( Snapshot rocksDBSnapshot = db.getSnapshot(); try ( - RocksIteratorWrapper iterator = getRocksIterator(db, stateInfo.f0); + RocksIteratorWrapper iterator = getRocksIterator(db, stateMetaInfo.f0); RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions()) ) { iterator.seekToFirst(); - @SuppressWarnings("unchecked") - TypeSerializerSnapshot priorValueSerializerSnapshot = (TypeSerializerSnapshot) - Preconditions.checkNotNull(restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)); - TypeSerializer priorValueSerializer = priorValueSerializerSnapshot.restoreSerializer(); - DataInputDeserializer serializedValueInput = new DataInputDeserializer(); DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512); while (iterator.isValid()) { @@ -1501,10 +1480,10 @@ private void migrateStateValues( rocksDBState.migrateSerializedValue( serializedValueInput, migratedSerializedValueOutput, - priorValueSerializer, - newStateSerializer); + stateMetaInfo.f1.getPreviousStateSerializer(), + stateMetaInfo.f1.getStateSerializer()); - batchWriter.put(stateInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer()); + batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer()); migratedSerializedValueOutput.clear(); iterator.next(); @@ -1698,25 +1677,16 @@ private Tuple2 tryRegisterP // TODO we implement the simple way of supporting the current functionality, mimicking keyed state // because this should be reworked in FLINK-9376 and then we should have a common algorithm over // StateMetaInfoSnapshot that avoids this code duplication. - StateMetaInfoSnapshot restoredMetaInfoSnapshot = restoredKvStateMetaInfos.get(stateName); - - Preconditions.checkState( - restoredMetaInfoSnapshot != null, - "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," + - " but its corresponding restored snapshot cannot be found."); - StateMetaInfoSnapshot.CommonSerializerKeys serializerKey = - StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER; - - TypeSerializer metaInfoTypeSerializer = restoredMetaInfoSnapshot.restoreTypeSerializer(serializerKey); + @SuppressWarnings("unchecked") + RegisteredPriorityQueueStateBackendMetaInfo castedMetaInfo = + (RegisteredPriorityQueueStateBackendMetaInfo) metaInfoTuple.f1; - if (metaInfoTypeSerializer != byteOrderedElementSerializer) { - @SuppressWarnings("unchecked") - TypeSerializerSnapshot serializerSnapshot = Preconditions.checkNotNull( - (TypeSerializerSnapshot) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(serializerKey)); + TypeSerializer previousElementSerializer = castedMetaInfo.getPreviousElementSerializer(); + if (previousElementSerializer != byteOrderedElementSerializer) { TypeSerializerSchemaCompatibility compatibilityResult = - serializerSnapshot.resolveSchemaCompatibility(byteOrderedElementSerializer); + castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer); // Since priority queue elements are written into RocksDB // as keys prefixed with the key group and namespace, we do not support diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java index 19d978322f09c..047d0b887b3d4 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java @@ -504,7 +504,9 @@ public boolean canEqual(Object obj) { } } - private static class UnionSerializer extends TypeSerializer> { + @VisibleForTesting + @Internal + static class UnionSerializer extends TypeSerializer> { private static final long serialVersionUID = 1L; private final TypeSerializer oneSerializer; @@ -522,12 +524,22 @@ public boolean isImmutableType() { @Override public TypeSerializer> duplicate() { - return this; + TypeSerializer duplicateOne = oneSerializer.duplicate(); + TypeSerializer duplicateTwo = twoSerializer.duplicate(); + + // compare reference of nested serializers, if same instances returned, we can reuse + // this instance as well + if (duplicateOne != oneSerializer || duplicateTwo != twoSerializer) { + return new UnionSerializer<>(duplicateOne, duplicateTwo); + } else { + return this; + } } @Override public TaggedUnion createInstance() { - return null; + //we arbitrarily always create instance of one + return TaggedUnion.one(oneSerializer.createInstance()); } @Override diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java index 271d9bec33f0e..a0af5f1a8c317 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java @@ -234,7 +234,9 @@ public final DataStream union(DataStream... streams) { * {@link org.apache.flink.streaming.api.collector.selector.OutputSelector} * for directing the tuples. * @return The {@link SplitStream} + * @deprecated Please use side ouput instead. */ + @Deprecated public SplitStream split(OutputSelector outputSelector) { return new SplitStream<>(this, clean(outputSelector)); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java index 0beae32435d4c..7f28dc7b497d6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java @@ -33,6 +33,7 @@ * @param The type of the elements in the Stream */ +@Deprecated @PublicEvolving public class SplitStream extends DataStream { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java index ed69ed4ab8467..9303166dbf943 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java @@ -77,6 +77,7 @@ import org.apache.flink.streaming.api.transformations.StreamTransformation; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SplittableIterator; +import org.apache.flink.util.StringUtils; import com.esotericsoftware.kryo.Serializer; @@ -958,8 +959,7 @@ public DataStreamSource readTextFile(String filePath) { * @return The data stream that represents the data read from the given file as text lines */ public DataStreamSource readTextFile(String filePath, String charsetName) { - Preconditions.checkNotNull(filePath, "The file path must not be null."); - Preconditions.checkNotNull(filePath.isEmpty(), "The file path must not be empty."); + Preconditions.checkArgument(!StringUtils.isNullOrWhitespaceOnly(filePath), "The file path must not be null or blank."); TextInputFormat format = new TextInputFormat(new Path(filePath)); format.setFilesFilter(FilePathFilter.createDefaultFilter()); @@ -1156,8 +1156,7 @@ public DataStreamSource readFile(FileInputFormat inputFormat, TypeInformation typeInformation) { Preconditions.checkNotNull(inputFormat, "InputFormat must not be null."); - Preconditions.checkNotNull(filePath, "The file path must not be null."); - Preconditions.checkNotNull(filePath.isEmpty(), "The file path must not be empty."); + Preconditions.checkArgument(!StringUtils.isNullOrWhitespaceOnly(filePath), "The file path must not be null or blank."); inputFormat.setFilePath(filePath); return createFileInput(inputFormat, typeInformation, "Custom File Source", watchType, interval); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java index 03f12b585ae9e..e39335479f3c5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java @@ -39,20 +39,24 @@ import org.apache.flink.runtime.state.FunctionInitializationContext; import org.apache.flink.runtime.state.FunctionSnapshotContext; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.util.FlinkRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.time.Clock; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; import static java.util.Objects.requireNonNull; import static org.apache.flink.util.Preconditions.checkArgument; @@ -150,6 +154,12 @@ protected TXN currentTransaction() { return currentTransactionHolder == null ? null : currentTransactionHolder.handle; } + @Nonnull + protected Stream> pendingTransactions() { + return pendingCommitTransactions.entrySet().stream() + .map(e -> new AbstractMap.SimpleEntry<>(e.getKey(), e.getValue().handle)); + } + // ------ methods that should be implemented in child class to support two phase commit algorithm ------ /** @@ -257,6 +267,7 @@ public final void notifyCheckpointComplete(long checkpointId) throws Exception { Iterator>> pendingTransactionIterator = pendingCommitTransactions.entrySet().iterator(); checkState(pendingTransactionIterator.hasNext(), "checkpoint completed, but no transaction pending"); + Throwable firstError = null; while (pendingTransactionIterator.hasNext()) { Map.Entry> entry = pendingTransactionIterator.next(); @@ -270,12 +281,23 @@ public final void notifyCheckpointComplete(long checkpointId) throws Exception { name(), checkpointId, pendingTransaction, pendingTransactionCheckpointId); logWarningIfTimeoutAlmostReached(pendingTransaction); - commit(pendingTransaction.handle); + try { + commit(pendingTransaction.handle); + } catch (Throwable t) { + if (firstError == null) { + firstError = t; + } + } LOG.debug("{} - committed checkpoint transaction {}", name(), pendingTransaction); pendingTransactionIterator.remove(); } + + if (firstError != null) { + throw new FlinkRuntimeException("Committing one of transactions failed, logging first encountered failure", + firstError); + } } @Override @@ -320,9 +342,9 @@ public void initializeState(FunctionInitializationContext context) throws Except state = context.getOperatorStateStore().getListState(stateDescriptor); + boolean recoveredUserContext = false; if (context.isRestored()) { LOG.info("{} - restoring state", name()); - for (State operatorState : state.get()) { userContext = operatorState.getContext(); List> recoveredTransactions = operatorState.getPendingCommitTransactions(); @@ -337,11 +359,13 @@ public void initializeState(FunctionInitializationContext context) throws Except if (userContext.isPresent()) { finishRecoveringContext(); + recoveredUserContext = true; } } } + // if in restore we didn't get any userContext or we are initializing from scratch - if (userContext == null) { + if (!recoveredUserContext) { LOG.info("{} - no state to restore", name()); userContext = initializeUserContext(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java index 65a7628578c39..3252d9c0225bd 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java @@ -21,8 +21,10 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; import org.apache.flink.core.fs.RecoverableWriter; -import org.apache.flink.util.Preconditions; +import org.apache.flink.core.fs.RecoverableWriter.CommitRecoverable; +import org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,11 +33,15 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Objects; +import java.util.TreeMap; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; /** * A bucket is the directory organization of the output of the {@link StreamingFileSink}. @@ -62,14 +68,16 @@ public class Bucket { private final RollingPolicy rollingPolicy; - private final Map> pendingPartsPerCheckpoint; + private final NavigableMap resumablesPerCheckpoint; + + private final NavigableMap> pendingPartsPerCheckpoint; private long partCounter; @Nullable private PartFileWriter inProgressPart; - private List pendingPartsForCurrentCheckpoint; + private List pendingPartsForCurrentCheckpoint; /** * Constructor to create a new empty bucket. @@ -83,16 +91,17 @@ private Bucket( final PartFileWriter.PartFileFactory partFileFactory, final RollingPolicy rollingPolicy) { - this.fsWriter = Preconditions.checkNotNull(fsWriter); + this.fsWriter = checkNotNull(fsWriter); this.subtaskIndex = subtaskIndex; - this.bucketId = Preconditions.checkNotNull(bucketId); - this.bucketPath = Preconditions.checkNotNull(bucketPath); + this.bucketId = checkNotNull(bucketId); + this.bucketPath = checkNotNull(bucketPath); this.partCounter = initialPartCounter; - this.partFileFactory = Preconditions.checkNotNull(partFileFactory); - this.rollingPolicy = Preconditions.checkNotNull(rollingPolicy); + this.partFileFactory = checkNotNull(partFileFactory); + this.rollingPolicy = checkNotNull(rollingPolicy); this.pendingPartsForCurrentCheckpoint = new ArrayList<>(); - this.pendingPartsPerCheckpoint = new HashMap<>(); + this.pendingPartsPerCheckpoint = new TreeMap<>(); + this.resumablesPerCheckpoint = new TreeMap<>(); } /** @@ -120,20 +129,34 @@ private Bucket( } private void restoreInProgressFile(final BucketState state) throws IOException { + if (!state.hasInProgressResumableFile()) { + return; + } // we try to resume the previous in-progress file - if (state.hasInProgressResumableFile()) { - final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); + final ResumeRecoverable resumable = state.getInProgressResumableFile(); + + if (fsWriter.supportsResume()) { + final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( - bucketId, fsWriter, resumable, state.getInProgressFileCreationTime()); + bucketId, stream, resumable, state.getInProgressFileCreationTime()); + } else { + // if the writer does not support resume, then we close the + // in-progress part and commit it, as done in the case of pending files. + + fsWriter.recoverForCommit(resumable).commitAfterRecovery(); + } + + if (fsWriter.requiresCleanupOfRecoverableState()) { + fsWriter.cleanupRecoverableState(resumable); } } private void commitRecoveredPendingFiles(final BucketState state) throws IOException { // we commit pending files for checkpoints that precess the last successful one, from which we are recovering - for (List committables: state.getCommittableFilesPerCheckpoint().values()) { - for (RecoverableWriter.CommitRecoverable committable: committables) { + for (List committables: state.getCommittableFilesPerCheckpoint().values()) { + for (CommitRecoverable committable: committables) { fsWriter.recoverForCommit(committable).commitAfterRecovery(); } } @@ -156,8 +179,8 @@ boolean isActive() { } void merge(final Bucket bucket) throws IOException { - Preconditions.checkNotNull(bucket); - Preconditions.checkState(Objects.equals(bucket.bucketPath, bucketPath)); + checkNotNull(bucket); + checkState(Objects.equals(bucket.bucketPath, bucketPath)); // There should be no pending files in the "to-merge" states. // The reason is that: @@ -165,10 +188,10 @@ void merge(final Bucket bucket) throws IOException { // So a snapshot, including the one we are recovering from, will never contain such files. // 2) the files in pendingPartsPerCheckpoint are committed upon recovery (see commitRecoveredPendingFiles()). - Preconditions.checkState(bucket.pendingPartsForCurrentCheckpoint.isEmpty()); - Preconditions.checkState(bucket.pendingPartsPerCheckpoint.isEmpty()); + checkState(bucket.pendingPartsForCurrentCheckpoint.isEmpty()); + checkState(bucket.pendingPartsPerCheckpoint.isEmpty()); - RecoverableWriter.CommitRecoverable committable = bucket.closePartFile(); + CommitRecoverable committable = bucket.closePartFile(); if (committable != null) { pendingPartsForCurrentCheckpoint.add(committable); } @@ -195,7 +218,8 @@ private void rollPartFile(final long currentTime) throws IOException { closePartFile(); final Path partFilePath = assembleNewPartPath(); - inProgressPart = partFileFactory.openNew(bucketId, fsWriter, partFilePath, currentTime); + final RecoverableFsDataOutputStream stream = fsWriter.open(partFilePath); + inProgressPart = partFileFactory.openNew(bucketId, stream, partFilePath, currentTime); if (LOG.isDebugEnabled()) { LOG.debug("Subtask {} opening new part file \"{}\" for bucket id={}.", @@ -209,8 +233,8 @@ private Path assembleNewPartPath() { return new Path(bucketPath, PART_PREFIX + '-' + subtaskIndex + '-' + partCounter); } - private RecoverableWriter.CommitRecoverable closePartFile() throws IOException { - RecoverableWriter.CommitRecoverable committable = null; + private CommitRecoverable closePartFile() throws IOException { + CommitRecoverable committable = null; if (inProgressPart != null) { committable = inProgressPart.closeForCommit(); pendingPartsForCurrentCheckpoint.add(committable); @@ -228,12 +252,21 @@ void disposePartFile() { BucketState onReceptionOfCheckpoint(long checkpointId) throws IOException { prepareBucketForCheckpointing(checkpointId); - RecoverableWriter.ResumeRecoverable inProgressResumable = null; + ResumeRecoverable inProgressResumable = null; long inProgressFileCreationTime = Long.MAX_VALUE; if (inProgressPart != null) { inProgressResumable = inProgressPart.persist(); inProgressFileCreationTime = inProgressPart.getCreationTime(); + + // the following is an optimization so that writers that do not + // require cleanup, they do not have to keep track of resumables + // and later iterate over the active buckets. + // (see onSuccessfulCompletionOfCheckpoint()) + + if (fsWriter.requiresCleanupOfRecoverableState()) { + this.resumablesPerCheckpoint.put(checkpointId, inProgressResumable); + } } return new BucketState<>(bucketId, bucketPath, inProgressFileCreationTime, inProgressResumable, pendingPartsPerCheckpoint); @@ -254,19 +287,36 @@ private void prepareBucketForCheckpointing(long checkpointId) throws IOException } void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { - Preconditions.checkNotNull(fsWriter); + checkNotNull(fsWriter); + + Iterator>> it = + pendingPartsPerCheckpoint.headMap(checkpointId, true) + .entrySet().iterator(); + + while (it.hasNext()) { + Map.Entry> entry = it.next(); + + for (CommitRecoverable committable : entry.getValue()) { + fsWriter.recoverForCommit(committable).commit(); + } + it.remove(); + } + + cleanupOutdatedResumables(checkpointId); + } - Iterator>> it = - pendingPartsPerCheckpoint.entrySet().iterator(); + private void cleanupOutdatedResumables(long checkpointId) throws IOException { + Iterator> it = + resumablesPerCheckpoint.headMap(checkpointId, false) + .entrySet().iterator(); while (it.hasNext()) { - Map.Entry> entry = it.next(); + final ResumeRecoverable recoverable = it.next().getValue(); + final boolean successfullyDeleted = fsWriter.cleanupRecoverableState(recoverable); + it.remove(); - if (entry.getKey() <= checkpointId) { - for (RecoverableWriter.CommitRecoverable committable : entry.getValue()) { - fsWriter.recoverForCommit(committable).commit(); - } - it.remove(); + if (LOG.isDebugEnabled() && successfullyDeleted) { + LOG.debug("Subtask {} successfully deleted incomplete part for bucket id={}.", subtaskIndex, bucketId); } } } @@ -285,7 +335,7 @@ void onProcessingTime(long timestamp) throws IOException { // --------------------------- Testing Methods ----------------------------- @VisibleForTesting - Map> getPendingPartsPerCheckpoint() { + Map> getPendingPartsPerCheckpoint() { return pendingPartsPerCheckpoint; } @@ -296,7 +346,7 @@ PartFileWriter getInProgressPart() { } @VisibleForTesting - List getPendingPartsForCurrentCheckpoint() { + List getPendingPartsForCurrentCheckpoint() { return pendingPartsForCurrentCheckpoint; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java index d08bc2ac0c382..7b35e50283065 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java @@ -254,7 +254,7 @@ private void snapshotActiveBuckets( } } - void onElement(final IN value, final SinkFunction.Context context) throws Exception { + Bucket onElement(final IN value, final SinkFunction.Context context) throws Exception { final long currentProcessingTime = context.currentProcessingTime(); // setting the values in the bucketer context @@ -272,6 +272,7 @@ void onElement(final IN value, final SinkFunction.Context context) throws Except // another part file for the bucket, if we start from 0 we may overwrite previous parts. this.maxPartCounter = Math.max(maxPartCounter, bucket.getPartCounter()); + return bucket; } private Bucket getOrCreateBucketForBucketId(final BucketID bucketId) throws IOException { @@ -304,7 +305,11 @@ void close() { } private Path assembleBucketPath(BucketID bucketId) { - return new Path(basePath, bucketId.toString()); + final String child = bucketId.toString(); + if ("".equals(child)) { + return basePath; + } + return new Path(basePath, child); } /** diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java index 005ae4e737fb9..a44b0e8aea3da 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java @@ -79,14 +79,13 @@ static class Factory implements PartFileWriter.PartFileFactory resumeFrom( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final RecoverableWriter.ResumeRecoverable resumable, final long creationTime) throws IOException { - Preconditions.checkNotNull(fileSystemWriter); + Preconditions.checkNotNull(stream); Preconditions.checkNotNull(resumable); - final RecoverableFsDataOutputStream stream = fileSystemWriter.recover(resumable); final BulkWriter writer = writerFactory.create(stream); return new BulkPartWriter<>(bucketId, stream, writer, creationTime); } @@ -94,14 +93,13 @@ public PartFileWriter resumeFrom( @Override public PartFileWriter openNew( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final Path path, final long creationTime) throws IOException { - Preconditions.checkNotNull(fileSystemWriter); + Preconditions.checkNotNull(stream); Preconditions.checkNotNull(path); - final RecoverableFsDataOutputStream stream = fileSystemWriter.open(path); final BulkWriter writer = writerFactory.create(stream); return new BulkPartWriter<>(bucketId, stream, writer, creationTime); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java index 662454ba200b3..95a2978a4c601 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java @@ -111,7 +111,7 @@ interface PartFileFactory { /** * Used upon recovery from a failure to recover a {@link PartFileWriter writer}. * @param bucketId the id of the bucket this writer is writing to. - * @param fileSystemWriter the filesystem-specific writer to use when writing to the filesystem. + * @param stream the filesystem-specific output stream to use when writing to the filesystem. * @param resumable the state of the stream we are resurrecting. * @param creationTime the creation time of the stream. * @return the recovered {@link PartFileWriter writer}. @@ -119,14 +119,14 @@ interface PartFileFactory { */ PartFileWriter resumeFrom( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final RecoverableWriter.ResumeRecoverable resumable, final long creationTime) throws IOException; /** * Used to create a new {@link PartFileWriter writer}. * @param bucketId the id of the bucket this writer is writing to. - * @param fileSystemWriter the filesystem-specific writer to use when writing to the filesystem. + * @param stream the filesystem-specific output stream to use when writing to the filesystem. * @param path the part this writer will write to. * @param creationTime the creation time of the stream. * @return the new {@link PartFileWriter writer}. @@ -134,7 +134,7 @@ PartFileWriter resumeFrom( */ PartFileWriter openNew( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final Path path, final long creationTime) throws IOException; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java index 2478b79a52797..05c160c262964 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java @@ -67,28 +67,26 @@ static class Factory implements PartFileWriter.PartFileFactory resumeFrom( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final RecoverableWriter.ResumeRecoverable resumable, final long creationTime) throws IOException { - Preconditions.checkNotNull(fileSystemWriter); + Preconditions.checkNotNull(stream); Preconditions.checkNotNull(resumable); - final RecoverableFsDataOutputStream stream = fileSystemWriter.recover(resumable); return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); } @Override public PartFileWriter openNew( final BucketID bucketId, - final RecoverableWriter fileSystemWriter, + final RecoverableFsDataOutputStream stream, final Path path, final long creationTime) throws IOException { - Preconditions.checkNotNull(fileSystemWriter); + Preconditions.checkNotNull(stream); Preconditions.checkNotNull(path); - final RecoverableFsDataOutputStream stream = fileSystemWriter.open(path); return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java index 2c4ae4a993dfa..82400036c3c83 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java @@ -258,6 +258,8 @@ private Collection transformSplit(SplitTransformation split) { StreamTransformation input = split.getInput(); Collection resultIds = transform(input); + validateSplitTransformation(input); + // the recursive transform call might have transformed this already if (alreadyTransformed.containsKey(split)) { return alreadyTransformed.get(split); @@ -643,4 +645,20 @@ private String determineSlotSharingGroup(String specifiedGroup, Collection void validateSplitTransformation(StreamTransformation input) { + if (input instanceof SelectTransformation || input instanceof SplitTransformation) { + throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + } else if (input instanceof SideOutputTransformation) { + throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); + } else if (input instanceof UnionTransformation) { + for (StreamTransformation transformation : ((UnionTransformation) input).getInputs()) { + validateSplitTransformation(transformation); + } + } else if (input instanceof PartitionTransformation) { + validateSplitTransformation(((PartitionTransformation) input).getInput()); + } else { + return; + } + } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java index d7c51bcbdeb16..a87f49fc7b8f2 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java @@ -177,7 +177,8 @@ private boolean generateNodeHash( for (byte[] previousHash : hashes.values()) { if (Arrays.equals(previousHash, hash)) { - throw new IllegalArgumentException("Hash collision on user-specified ID. " + + throw new IllegalArgumentException("Hash collision on user-specified ID " + + "\"" + userSpecifiedHash + "\". " + "Most likely cause is a non-unique ID. Please check that all IDs " + "specified via `uid(String)` are unique."); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java index a63a7971679d6..d9a195cb23553 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java @@ -413,8 +413,13 @@ public final OperatorSnapshotFutures snapshotState(long checkpointId, long times snapshotException.addSuppressed(e); } - throw new Exception("Could not complete snapshot " + checkpointId + " for operator " + - getOperatorName() + '.', snapshotException); + String snapshotFailMessage = "Could not complete snapshot " + checkpointId + " for operator " + + getOperatorName() + "."; + + if (!getContainingTask().isCanceled()) { + LOG.info(snapshotFailMessage, snapshotException); + } + throw new Exception(snapshotFailMessage, snapshotException); } return snapshotInProgress; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java index 63dd3e4d42740..f09d83e45119b 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java @@ -155,7 +155,7 @@ public LatencyMarksEmitter( public void onProcessingTime(long timestamp) throws Exception { try { // ProcessingTimeService callbacks are executed under the checkpointing lock - output.emitLatencyMarker(new LatencyMarker(timestamp, operatorId, subtaskIndex)); + output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex)); } catch (Throwable t) { // we catch the Throwables here so that we don't trigger the processing // timer services async exception handler diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index aa6774dab8fc9..215e5485648f4 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -79,6 +79,7 @@ import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; import org.apache.flink.util.TestLogger; import org.hamcrest.core.StringStartsWith; @@ -952,12 +953,7 @@ public boolean filter(Integer value) throws Exception { fail(e.getMessage()); } - OutputSelector outputSelector = new OutputSelector() { - @Override - public Iterable select(Integer value) { - return null; - } - }; + OutputSelector outputSelector = new DummyOutputSelector<>(); SplitStream split = unionFilter.split(outputSelector); split.select("dummy").addSink(new DiscardingSink()); @@ -1091,6 +1087,91 @@ public void testChannelSelectors() { assertTrue(globalPartitioner instanceof GlobalPartitioner); } + ///////////////////////////////////////////////////////////// + // Split testing + ///////////////////////////////////////////////////////////// + + @Test + public void testConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testSplitAfterSideOutputRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputTag outputTag = new OutputTag("dummy"){}; + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.getSideOutput(outputTag).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testSelectBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testUnionBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").union(src.map(x -> x)).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testKeybyBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").keyBy(x -> x).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + ///////////////////////////////////////////////////////////// // KeyBy testing ///////////////////////////////////////////////////////////// @@ -1427,4 +1508,11 @@ public int getI() { return i; } } + + private class DummyOutputSelector implements OutputSelector { + @Override + public Iterable select(Integer value) { + return null; + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java index fc8e9971683f9..3bf1573d6e420 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java @@ -18,11 +18,18 @@ package org.apache.flink.streaming.api.datastream; import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.state.CheckpointListener; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; @@ -39,7 +46,6 @@ import java.io.BufferedOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -64,15 +70,18 @@ public class ReinterpretDataStreamAsKeyedStreamITCase { @Test public void testReinterpretAsKeyedStream() throws Exception { - final int numEventsPerInstance = 100; final int maxParallelism = 8; + final int numEventsPerInstance = 100; final int parallelism = 3; - final int numUniqueKeys = 12; + final int numTotalEvents = numEventsPerInstance * parallelism; + final int numUniqueKeys = 100; final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); env.setMaxParallelism(maxParallelism); env.setParallelism(parallelism); + env.enableCheckpointing(100); + env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L)); final List partitionFiles = new ArrayList<>(parallelism); for (int i = 0; i < parallelism; ++i) { @@ -93,7 +102,7 @@ public void testReinterpretAsKeyedStream() throws Exception { .timeWindow(Time.seconds(1)) // test that also timers and aggregated state work as expected .reduce((ReduceFunction>) (value1, value2) -> new Tuple2<>(value1.f0, value1.f1 + value2.f1)) - .addSink(new ValidatingSink(numEventsPerInstance * parallelism)).setParallelism(1); + .addSink(new ValidatingSink(numTotalEvents)).setParallelism(1); env.execute(); } @@ -101,10 +110,10 @@ public void testReinterpretAsKeyedStream() throws Exception { private static class RandomTupleSource implements ParallelSourceFunction> { private static final long serialVersionUID = 1L; - private int numKeys; + private final int numKeys; private int remainingEvents; - public RandomTupleSource(int numEvents, int numKeys) { + RandomTupleSource(int numEvents, int numKeys) { this.numKeys = numKeys; this.remainingEvents = numEvents; } @@ -112,8 +121,11 @@ public RandomTupleSource(int numEvents, int numKeys) { @Override public void run(SourceContext> out) throws Exception { Random random = new Random(42); - while (--remainingEvents >= 0) { - out.collect(new Tuple2<>(random.nextInt(numKeys), 1)); + while (remainingEvents > 0) { + synchronized (out.getCheckpointLock()) { + out.collect(new Tuple2<>(random.nextInt(numKeys), 1)); + --remainingEvents; + } } } @@ -130,7 +142,7 @@ private static class ToPartitionFileSink extends RichSinkFunction allPartitions; private DataOutputStream dos; - public ToPartitionFileSink(List allPartitions) { + ToPartitionFileSink(List allPartitions) { this.allPartitions = allPartitions; } @@ -156,24 +168,41 @@ public void invoke(Tuple2 value, Context context) throws Excep } } - private static class FromPartitionFileSource extends RichParallelSourceFunction> { + private static class FromPartitionFileSource extends RichParallelSourceFunction> + implements CheckpointedFunction, CheckpointListener { private static final long serialVersionUID = 1L; - private List allPartitions; + private final List allPartitions; private DataInputStream din; private volatile boolean running; - public FromPartitionFileSource(List allPartitons) { - this.allPartitions = allPartitons; + private long fileLength; + private long waitForFailurePos; + private long position; + private transient ListState positionState; + private transient boolean isRestored; + + private transient volatile boolean canFail; + + FromPartitionFileSource(List allPartitions) { + this.allPartitions = allPartitions; } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); + File partitionFile = allPartitions.get(subtaskIdx); + fileLength = partitionFile.length(); + waitForFailurePos = fileLength * 3 / 4; din = new DataInputStream( new BufferedInputStream( - new FileInputStream(allPartitions.get(subtaskIdx)))); + new FileInputStream(partitionFile))); + + long toSkip = position; + while (toSkip > 0L) { + toSkip -= din.skip(toSkip); + } } @Override @@ -184,29 +213,78 @@ public void close() throws Exception { @Override public void run(SourceContext> out) throws Exception { - this.running = true; - try { - while (running) { + + running = true; + + while (running && hasMoreDataToRead()) { + + synchronized (out.getCheckpointLock()) { Integer key = din.readInt(); Integer val = din.readInt(); out.collect(new Tuple2<>(key, val)); + + position += 2 * Integer.BYTES; + } + + if (shouldWaitForCompletedCheckpointAndFailNow()) { + while (!canFail) { + // wait for a checkpoint to complete + Thread.sleep(10L); + } + throw new Exception("Artificial failure."); } - } catch (EOFException ignore) { } } + private boolean shouldWaitForCompletedCheckpointAndFailNow() { + return !isRestored && position > waitForFailurePos; + } + + private boolean hasMoreDataToRead() { + return position < fileLength; + } + @Override public void cancel() { this.running = false; } + + @Override + public void notifyCheckpointComplete(long checkpointId) { + canFail = !isRestored; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) throws Exception { + positionState.clear(); + positionState.add(position); + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + canFail = false; + position = 0L; + isRestored = context.isRestored(); + positionState = context.getOperatorStateStore().getListState( + new ListStateDescriptor<>("posState", Long.class)); + + if (isRestored) { + for (long value : positionState.get()) { + position += value; + } + } + } } - private static class ValidatingSink extends RichSinkFunction> { + private static class ValidatingSink extends RichSinkFunction> + implements CheckpointedFunction { private static final long serialVersionUID = 1L; private final int expectedSum; private int runningSum = 0; + private transient ListState sumState; + private ValidatingSink(int expectedSum) { this.expectedSum = expectedSum; } @@ -227,5 +305,23 @@ public void close() throws Exception { Assert.assertEquals(expectedSum, runningSum); super.close(); } + + @Override + public void snapshotState(FunctionSnapshotContext context) throws Exception { + sumState.clear(); + sumState.add(runningSum); + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + sumState = context.getOperatorStateStore().getListState( + new ListStateDescriptor<>("sumState", Integer.class)); + + if (context.isRestored()) { + for (int value : sumState.get()) { + runningSum += value; + } + } + } } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/UnionSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/UnionSerializerTest.java new file mode 100644 index 0000000000000..29304bcfc8770 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/UnionSerializerTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.datastream; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeutils.SerializerTestBase; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; +import org.apache.flink.streaming.api.datastream.CoGroupedStreams.TaggedUnion; +import org.apache.flink.streaming.api.datastream.CoGroupedStreams.UnionSerializer; +import org.apache.flink.testutils.DeeplyEqualsChecker; + +/** + * Serializer tests for {@link UnionSerializer}. + */ +public class UnionSerializerTest extends SerializerTestBase> { + + public UnionSerializerTest() { + super(new DeeplyEqualsChecker() + .withCustomCheck( + (o1, o2) -> o1 instanceof TaggedUnion && o2 instanceof TaggedUnion, + (o1, o2, checker) -> { + TaggedUnion union1 = (TaggedUnion) o1; + TaggedUnion union2 = (TaggedUnion) o2; + + if (union1.isOne() && union2.isOne()) { + return checker.deepEquals(union1.getOne(), union2.getOne()); + } else if (union1.isTwo() && union2.isTwo()) { + return checker.deepEquals(union1.getTwo(), union2.getTwo()); + } else { + return false; + } + } + )); + } + + @Override + protected TypeSerializer> createSerializer() { + return new UnionSerializer<>( + new KryoSerializer<>(Object.class, new ExecutionConfig()), + new KryoSerializer<>(Object.class, new ExecutionConfig()) + ); + } + + @Override + protected int getLength() { + return -1; + } + + @Override + @SuppressWarnings("unchecked") + protected Class> getTypeClass() { + return (Class>) (Class) TaggedUnion.class; + } + + @Override + @SuppressWarnings("unchecked") + protected TaggedUnion[] getTestData() { + return new TaggedUnion[]{ + TaggedUnion.one(1), + TaggedUnion.two("A"), + TaggedUnion.one("C") + }; + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/SocketClientSinkTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/SocketClientSinkTest.java index b3c4ee93ede60..7c9863ef07ff5 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/SocketClientSinkTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/SocketClientSinkTest.java @@ -24,11 +24,13 @@ import org.apache.flink.util.TestLogger; import org.apache.commons.io.IOUtils; +import org.junit.AssumptionViolatedException; import org.junit.Test; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.BindException; import java.net.ServerSocket; import java.net.Socket; import java.util.concurrent.Callable; @@ -279,7 +281,12 @@ public Void call() throws Exception { retryLatch.countDown(); // Restart the server - serverSocket[0] = new ServerSocket(port); + try { + serverSocket[0] = new ServerSocket(port); + } catch (BindException be) { + // some other process may be using this port now + throw new AssumptionViolatedException("Could not bind server to previous port.", be); + } Socket socket = serverSocket[0].accept(); BufferedReader reader = new BufferedReader(new InputStreamReader( diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java new file mode 100644 index 0000000000000..5ae57ce43bc21 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; + +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +/** + * Integration tests for {@link BucketAssigner bucket assigners}. + */ +public class BucketAssignerITCases { + + @ClassRule + public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); + + @Test + public void testAssembleBucketPath() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + final Path basePath = new Path(outDir.toURI()); + final long time = 1000L; + + final RollingPolicy rollingPolicy = + DefaultRollingPolicy + .create() + .withMaxPartSize(7L) + .build(); + + final Buckets buckets = new Buckets<>( + basePath, + new BasePathBucketAssigner<>(), + new DefaultBucketFactoryImpl<>(), + new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + rollingPolicy, + 0 + ); + + Bucket bucket = + buckets.onElement("abc", new TestUtils.MockSinkContext(time, time, time)); + Assert.assertEquals(new Path(basePath.toUri()), bucket.getBucketPath()); + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java new file mode 100644 index 0000000000000..308bc31f1848e --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java @@ -0,0 +1,446 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; +import org.apache.flink.core.fs.local.LocalFileSystem; +import org.apache.flink.core.fs.local.LocalRecoverableWriter; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; +import org.apache.flink.streaming.api.functions.sink.filesystem.utils.NoOpCommitter; +import org.apache.flink.streaming.api.functions.sink.filesystem.utils.NoOpRecoverable; +import org.apache.flink.streaming.api.functions.sink.filesystem.utils.NoOpRecoverableFsDataOutputStream; +import org.apache.flink.streaming.api.functions.sink.filesystem.utils.NoOpRecoverableWriter; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for the {@code Bucket}. + */ +public class BucketTest { + + @ClassRule + public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); + + @Test + public void shouldNotCleanupResumablesThatArePartOfTheAckedCheckpoint() throws IOException { + final File outDir = TEMP_FOLDER.newFolder(); + final Path path = new Path(outDir.toURI()); + + final TestRecoverableWriter recoverableWriter = getRecoverableWriter(path); + final Bucket bucketUnderTest = + createBucket(recoverableWriter, path, 0, 0); + + bucketUnderTest.write("test-element", 0L); + + final BucketState state = bucketUnderTest.onReceptionOfCheckpoint(0L); + assertThat(state, hasActiveInProgressFile()); + + bucketUnderTest.onSuccessfulCompletionOfCheckpoint(0L); + assertThat(recoverableWriter, hasCalledDiscard(0)); // it did not discard as this is still valid. + } + + @Test + public void shouldCleanupOutdatedResumablesOnCheckpointAck() throws IOException { + final File outDir = TEMP_FOLDER.newFolder(); + final Path path = new Path(outDir.toURI()); + + final TestRecoverableWriter recoverableWriter = getRecoverableWriter(path); + final Bucket bucketUnderTest = + createBucket(recoverableWriter, path, 0, 0); + + bucketUnderTest.write("test-element", 0L); + + final BucketState state = bucketUnderTest.onReceptionOfCheckpoint(0L); + assertThat(state, hasActiveInProgressFile()); + + bucketUnderTest.onSuccessfulCompletionOfCheckpoint(0L); + + bucketUnderTest.onReceptionOfCheckpoint(1L); + bucketUnderTest.onReceptionOfCheckpoint(2L); + + bucketUnderTest.onSuccessfulCompletionOfCheckpoint(2L); + assertThat(recoverableWriter, hasCalledDiscard(2)); // that is for checkpoints 0 and 1 + } + + @Test + public void shouldCleanupResumableAfterRestoring() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + final Path path = new Path(outDir.toURI()); + + final TestRecoverableWriter recoverableWriter = getRecoverableWriter(path); + final Bucket bucketUnderTest = + createBucket(recoverableWriter, path, 0, 0); + + bucketUnderTest.write("test-element", 0L); + + final BucketState state = bucketUnderTest.onReceptionOfCheckpoint(0L); + assertThat(state, hasActiveInProgressFile()); + + bucketUnderTest.onSuccessfulCompletionOfCheckpoint(0L); + + final TestRecoverableWriter newRecoverableWriter = getRecoverableWriter(path); + restoreBucket(newRecoverableWriter, 0, 1, state); + + assertThat(newRecoverableWriter, hasCalledDiscard(1)); // that is for checkpoints 0 and 1 + } + + @Test + public void shouldNotCallCleanupWithoutInProgressPartFiles() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + final Path path = new Path(outDir.toURI()); + + final TestRecoverableWriter recoverableWriter = getRecoverableWriter(path); + final Bucket bucketUnderTest = + createBucket(recoverableWriter, path, 0, 0); + + final BucketState state = bucketUnderTest.onReceptionOfCheckpoint(0L); + assertThat(state, hasNoActiveInProgressFile()); + + bucketUnderTest.onReceptionOfCheckpoint(1L); + bucketUnderTest.onReceptionOfCheckpoint(2L); + + bucketUnderTest.onSuccessfulCompletionOfCheckpoint(2L); + assertThat(recoverableWriter, hasCalledDiscard(0)); // we have no in-progress file. + } + + // --------------------------- Checking Restore --------------------------- + + @Test + public void inProgressFileShouldBeCommittedIfWriterDoesNotSupportResume() throws IOException { + final StubNonResumableWriter nonResumableWriter = new StubNonResumableWriter(); + final Bucket bucket = getRestoredBucketWithOnlyInProgressPart(nonResumableWriter); + + Assert.assertThat(nonResumableWriter, hasMethodCallCountersEqualTo(1, 0, 1)); + Assert.assertThat(bucket, hasNullInProgressFile(true)); + } + + @Test + public void inProgressFileShouldBeRestoredIfWriterSupportsResume() throws IOException { + final StubResumableWriter resumableWriter = new StubResumableWriter(); + final Bucket bucket = getRestoredBucketWithOnlyInProgressPart(resumableWriter); + + Assert.assertThat(resumableWriter, hasMethodCallCountersEqualTo(1, 1, 0)); + Assert.assertThat(bucket, hasNullInProgressFile(false)); + } + + @Test + public void pendingFilesShouldBeRestored() throws IOException { + final int expectedRecoverForCommitCounter = 10; + + final StubNonResumableWriter writer = new StubNonResumableWriter(); + final Bucket bucket = getRestoredBucketWithOnlyPendingParts(writer, expectedRecoverForCommitCounter); + + Assert.assertThat(writer, hasMethodCallCountersEqualTo(0, 0, expectedRecoverForCommitCounter)); + Assert.assertThat(bucket, hasNullInProgressFile(true)); + } + + // ------------------------------- Matchers -------------------------------- + + private static TypeSafeMatcher hasCalledDiscard(int times) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(TestRecoverableWriter writer) { + return writer.getCleanupCallCounter() == times; + } + + @Override + public void describeTo(Description description) { + description + .appendText("the TestRecoverableWriter to have called discardRecoverableState() ") + .appendValue(times) + .appendText(" times."); + } + }; + } + + private static TypeSafeMatcher> hasActiveInProgressFile() { + return new TypeSafeMatcher>() { + @Override + protected boolean matchesSafely(BucketState state) { + return state.getInProgressResumableFile() != null; + } + + @Override + public void describeTo(Description description) { + description.appendText("a BucketState with active in-progress file."); + } + }; + } + + private static TypeSafeMatcher> hasNoActiveInProgressFile() { + return new TypeSafeMatcher>() { + @Override + protected boolean matchesSafely(BucketState state) { + return state.getInProgressResumableFile() == null; + } + + @Override + public void describeTo(Description description) { + description.appendText("a BucketState with no active in-progress file."); + } + }; + } + + private static TypeSafeMatcher> hasNullInProgressFile(final boolean isNull) { + + return new TypeSafeMatcher>() { + @Override + protected boolean matchesSafely(Bucket bucket) { + final PartFileWriter inProgressPart = bucket.getInProgressPart(); + return isNull == (inProgressPart == null); + } + + @Override + public void describeTo(Description description) { + description.appendText("a Bucket with its inProgressPart being ") + .appendText(isNull ? " null." : " not null."); + } + }; + } + + private static TypeSafeMatcher hasMethodCallCountersEqualTo( + final int supportsResumeCalls, + final int recoverCalls, + final int recoverForCommitCalls) { + + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(BaseStubWriter writer) { + return writer.getSupportsResumeCallCounter() == supportsResumeCalls && + writer.getRecoverCallCounter() == recoverCalls && + writer.getRecoverForCommitCallCounter() == recoverForCommitCalls; + } + + @Override + public void describeTo(Description description) { + description.appendText("a Writer where:") + .appendText(" supportsResume was called ").appendValue(supportsResumeCalls).appendText(" times,") + .appendText(" recover was called ").appendValue(recoverCalls).appendText(" times,") + .appendText(" and recoverForCommit was called ").appendValue(recoverForCommitCalls).appendText(" times.") + .appendText("'"); + } + }; + } + + // ------------------------------- Mock Classes -------------------------------- + + private static class TestRecoverableWriter extends LocalRecoverableWriter { + + private int cleanupCallCounter = 0; + + TestRecoverableWriter(LocalFileSystem fs) { + super(fs); + } + + int getCleanupCallCounter() { + return cleanupCallCounter; + } + + @Override + public boolean requiresCleanupOfRecoverableState() { + // here we return true so that the cleanupRecoverableState() is called. + return true; + } + + @Override + public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { + cleanupCallCounter++; + return false; + } + + @Override + public String toString() { + return "TestRecoverableWriter has called discardRecoverableState() " + cleanupCallCounter + " times."; + } + } + + /** + * A test implementation of a {@link RecoverableWriter} that does not support + * resuming, i.e. keep on writing to the in-progress file at the point we were + * before the failure. + */ + private static class StubResumableWriter extends BaseStubWriter { + + StubResumableWriter() { + super(true); + } + } + + /** + * A test implementation of a {@link RecoverableWriter} that does not support + * resuming, i.e. keep on writing to the in-progress file at the point we were + * before the failure. + */ + private static class StubNonResumableWriter extends BaseStubWriter { + + StubNonResumableWriter() { + super(false); + } + } + + /** + * A test implementation of a {@link RecoverableWriter} that does not support + * resuming, i.e. keep on writing to the in-progress file at the point we were + * before the failure. + */ + private static class BaseStubWriter extends NoOpRecoverableWriter { + + private final boolean supportsResume; + + private int supportsResumeCallCounter = 0; + + private int recoverCallCounter = 0; + + private int recoverForCommitCallCounter = 0; + + private BaseStubWriter(final boolean supportsResume) { + this.supportsResume = supportsResume; + } + + int getSupportsResumeCallCounter() { + return supportsResumeCallCounter; + } + + int getRecoverCallCounter() { + return recoverCallCounter; + } + + int getRecoverForCommitCallCounter() { + return recoverForCommitCallCounter; + } + + @Override + public RecoverableFsDataOutputStream recover(ResumeRecoverable resumable) throws IOException { + recoverCallCounter++; + return new NoOpRecoverableFsDataOutputStream(); + } + + @Override + public RecoverableFsDataOutputStream.Committer recoverForCommit(CommitRecoverable resumable) throws IOException { + checkArgument(resumable instanceof NoOpRecoverable); + recoverForCommitCallCounter++; + return new NoOpCommitter(); + } + + @Override + public boolean supportsResume() { + supportsResumeCallCounter++; + return supportsResume; + } + } + + // ------------------------------- Utility Methods -------------------------------- + + private static final String bucketId = "testing-bucket"; + + private static final RollingPolicy rollingPolicy = DefaultRollingPolicy.create().build(); + + private static final PartFileWriter.PartFileFactory partFileFactory = + new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()); + + private static Bucket createBucket( + final RecoverableWriter writer, + final Path bucketPath, + final int subtaskIdx, + final int initialPartCounter) { + + return Bucket.getNew( + writer, + subtaskIdx, + bucketId, + bucketPath, + initialPartCounter, + partFileFactory, + rollingPolicy); + } + + private static Bucket restoreBucket( + final RecoverableWriter writer, + final int subtaskIndex, + final long initialPartCounter, + final BucketState bucketState) throws Exception { + + return Bucket.restore( + writer, + subtaskIndex, + initialPartCounter, + partFileFactory, + rollingPolicy, + bucketState + ); + } + + private static TestRecoverableWriter getRecoverableWriter(Path path) { + try { + final FileSystem fs = FileSystem.get(path.toUri()); + if (!(fs instanceof LocalFileSystem)) { + fail("Expected Local FS but got a " + fs.getClass().getName() + " for path: " + path); + } + return new TestRecoverableWriter((LocalFileSystem) fs); + } catch (IOException e) { + fail(); + } + return null; + } + + private Bucket getRestoredBucketWithOnlyInProgressPart(final BaseStubWriter writer) throws IOException { + final BucketState stateWithOnlyInProgressFile = + new BucketState<>("test", new Path(), 12345L, new NoOpRecoverable(), new HashMap<>()); + return Bucket.restore(writer, 0, 1L, partFileFactory, rollingPolicy, stateWithOnlyInProgressFile); + } + + private Bucket getRestoredBucketWithOnlyPendingParts(final BaseStubWriter writer, final int numberOfPendingParts) throws IOException { + final Map> completePartsPerCheckpoint = + createPendingPartsPerCheckpoint(numberOfPendingParts); + + final BucketState initStateWithOnlyInProgressFile = + new BucketState<>("test", new Path(), 12345L, null, completePartsPerCheckpoint); + return Bucket.restore(writer, 0, 1L, partFileFactory, rollingPolicy, initStateWithOnlyInProgressFile); + } + + private Map> createPendingPartsPerCheckpoint(int noOfCheckpoints) { + final Map> pendingCommittablesPerCheckpoint = new HashMap<>(); + for (int checkpointId = 0; checkpointId < noOfCheckpoints; checkpointId++) { + final List pending = new ArrayList<>(); + pending.add(new NoOpRecoverable()); + pendingCommittablesPerCheckpoint.put((long) checkpointId, pending); + } + return pendingCommittablesPerCheckpoint; + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java index aee362178a76e..8369b90ac9665 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java @@ -188,7 +188,7 @@ public void testOnProcessingTime() throws Exception { // it takes the current processing time of the context for the creation time, // and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); // now it should roll buckets.onProcessingTime(7L); @@ -214,13 +214,13 @@ public void testBucketIsRemovedWhenNotActive() throws Exception { final Path path = new Path(outDir.toURI()); final OnProcessingTimePolicy rollOnProcessingTimeCountingPolicy = - new OnProcessingTimePolicy<>(2L); + new OnProcessingTimePolicy<>(2L); final Buckets buckets = createBuckets(path, rollOnProcessingTimeCountingPolicy, 0); // it takes the current processing time of the context for the creation time, and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); // now it should roll buckets.onProcessingTime(7L); @@ -244,7 +244,7 @@ public void testPartCounterAfterBucketResurrection() throws Exception { createBuckets(path, rollOnProcessingTimeCountingPolicy, 0); // it takes the current processing time of the context for the creation time, and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); Assert.assertEquals(1L, buckets.getActiveBuckets().get("test").getPartCounter()); // now it should roll @@ -257,7 +257,7 @@ public void testPartCounterAfterBucketResurrection() throws Exception { Assert.assertTrue(buckets.getActiveBuckets().isEmpty()); - buckets.onElement("test", new TestUtils.MockSinkContext(2L, 3L , 4L)); + buckets.onElement("test", new TestUtils.MockSinkContext(2L, 3L, 4L)); Assert.assertEquals(2L, buckets.getActiveBuckets().get("test").getPartCounter()); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpCommitter.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpCommitter.java new file mode 100644 index 0000000000000..06005a15a4869 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpCommitter.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem.utils; + +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; + +import java.io.IOException; + +/** + * An implementation of the {@link RecoverableFsDataOutputStream.Committer committer} + * that does nothing. + * + *

    This is to avoid to have to implement all methods for every implementation + * used in tests. + */ +public class NoOpCommitter implements RecoverableFsDataOutputStream.Committer { + + @Override + public void commit() throws IOException { + + } + + @Override + public void commitAfterRecovery() throws IOException { + + } + + @Override + public RecoverableWriter.CommitRecoverable getRecoverable() { + return null; + } +} diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3ConfigOptions.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverable.java similarity index 66% rename from flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3ConfigOptions.java rename to flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverable.java index 421880554e5e8..e00d4c4b0a9e0 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3ConfigOptions.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverable.java @@ -16,15 +16,17 @@ * limitations under the License. */ -package org.apache.flink.fs.s3.common.writer; +package org.apache.flink.streaming.api.functions.sink.filesystem.utils; + +import org.apache.flink.core.fs.RecoverableWriter; /** - * Configuration keys for the S3 file system based using Hadoop's s3a. + * An implementation of the {@link RecoverableWriter.ResumeRecoverable ResumeRecoverable} + * that does nothing. + * + *

    This is to avoid to have to implement all methods for every implementation + * used in tests. */ -public final class S3ConfigOptions { - - // ------------------------------------------------------------------------ +public class NoOpRecoverable implements RecoverableWriter.ResumeRecoverable { - /** Not meant to be instantiated. */ - private S3ConfigOptions() {} } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableFsDataOutputStream.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableFsDataOutputStream.java new file mode 100644 index 0000000000000..a549896922daf --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableFsDataOutputStream.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem.utils; + +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; + +import java.io.IOException; + +/** + * A default implementation of the {@link RecoverableFsDataOutputStream} that does nothing. + * + *

    This is to avoid to have to implement all methods for every implementation + * used in tests. + */ +public class NoOpRecoverableFsDataOutputStream extends RecoverableFsDataOutputStream { + @Override + public RecoverableWriter.ResumeRecoverable persist() throws IOException { + return null; + } + + @Override + public Committer closeForCommit() throws IOException { + return null; + } + + @Override + public void close() throws IOException { + + } + + @Override + public long getPos() throws IOException { + return 0; + } + + @Override + public void flush() throws IOException { + + } + + @Override + public void sync() throws IOException { + + } + + @Override + public void write(int b) throws IOException { + + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java new file mode 100644 index 0000000000000..e21da2aeabd73 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem.utils; + +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.IOException; + +/** + * A default implementation of the {@link RecoverableWriter} that does nothing. + * + *

    This is to avoid to have to implement all methods for every implementation + * used in tests. + */ +public class NoOpRecoverableWriter implements RecoverableWriter { + + @Override + public RecoverableFsDataOutputStream open(Path path) throws IOException { + return null; + } + + @Override + public RecoverableFsDataOutputStream recover(RecoverableWriter.ResumeRecoverable resumable) throws IOException { + return null; + } + + @Override + public boolean requiresCleanupOfRecoverableState() { + return false; + } + + @Override + public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public RecoverableFsDataOutputStream.Committer recoverForCommit(RecoverableWriter.CommitRecoverable resumable) throws IOException { + return null; + } + + @Override + public SimpleVersionedSerializer getCommitRecoverableSerializer() { + return null; + } + + @Override + public SimpleVersionedSerializer getResumeRecoverableSerializer() { + return null; + } + + @Override + public boolean supportsResume() { + return false; + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorLatencyMetricsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorLatencyMetricsTest.java index 14d51474b5753..dbcfc0df7d310 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorLatencyMetricsTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorLatencyMetricsTest.java @@ -175,6 +175,7 @@ private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOper output.size()); long timestamp = 0L; + int expectedLatencyIndex = 0; int i = 0; // verify that its only latency markers + a final watermark @@ -183,7 +184,14 @@ private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOper Assert.assertTrue(se.isLatencyMarker()); Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId()); Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex()); - Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp); + + // determines the next latency mark that should've been emitted + // latency marks are emitted once per latencyMarkInterval, + // as a result of which we never emit both 10 and 11 + while (timestamp > processingTimes.get(expectedLatencyIndex)) { + expectedLatencyIndex++; + } + Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime()); timestamp += latencyMarkInterval; } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorWatermarksTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorWatermarksTest.java index 4b5259e91c41d..00c3ac4ee7141 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorWatermarksTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorWatermarksTest.java @@ -101,7 +101,6 @@ public void testNoMaxWatermarkOnImmediateCancel() throws Exception { public void testNoMaxWatermarkOnAsyncCancel() throws Exception { final List output = new ArrayList<>(); - final Thread runner = Thread.currentThread(); // regular stream source operator final StreamSource> operator = @@ -117,7 +116,6 @@ public void run() { Thread.sleep(200); } catch (InterruptedException ignored) {} operator.cancel(); - runner.interrupt(); } }.start(); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockStreamingRuntimeContext.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockStreamingRuntimeContext.java index d024c04f5e018..655fe092823d8 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockStreamingRuntimeContext.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockStreamingRuntimeContext.java @@ -25,6 +25,8 @@ import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; +import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService; import java.util.Collections; @@ -79,6 +81,8 @@ public int getNumberOfParallelSubtasks() { private static class MockStreamOperator extends AbstractStreamOperator { private static final long serialVersionUID = -1153976702711944427L; + private transient TestProcessingTimeService testProcessingTimeService; + @Override public ExecutionConfig getExecutionConfig() { return new ExecutionConfig(); @@ -88,5 +92,13 @@ public ExecutionConfig getExecutionConfig() { public OperatorID getOperatorID() { return new OperatorID(); } + + @Override + protected ProcessingTimeService getProcessingTimeService() { + if (testProcessingTimeService == null) { + testProcessingTimeService = new TestProcessingTimeService(); + } + return testProcessingTimeService; + } } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java index 1b5113da2cbbf..53627d5ad4936 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java @@ -20,7 +20,7 @@ import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.runtime.checkpoint.StateObjectCollection; -import org.apache.flink.runtime.checkpoint.savepoint.SavepointV1Serializer; +import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2Serializer; import org.apache.flink.runtime.state.KeyedStateHandle; import org.apache.flink.runtime.state.OperatorStateHandle; @@ -55,13 +55,13 @@ public static void writeStateHandle(OperatorSubtaskState state, String path) thr dos.writeInt(0); // still required for compatibility - SavepointV1Serializer.serializeStreamStateHandle(null, dos); + SavepointV2Serializer.serializeStreamStateHandle(null, dos); Collection rawOperatorState = state.getRawOperatorState(); if (rawOperatorState != null) { dos.writeInt(rawOperatorState.size()); for (OperatorStateHandle operatorStateHandle : rawOperatorState) { - SavepointV1Serializer.serializeOperatorStateHandle(operatorStateHandle, dos); + SavepointV2Serializer.serializeOperatorStateHandle(operatorStateHandle, dos); } } else { // this means no states, not even an empty list @@ -72,7 +72,7 @@ public static void writeStateHandle(OperatorSubtaskState state, String path) thr if (managedOperatorState != null) { dos.writeInt(managedOperatorState.size()); for (OperatorStateHandle operatorStateHandle : managedOperatorState) { - SavepointV1Serializer.serializeOperatorStateHandle(operatorStateHandle, dos); + SavepointV2Serializer.serializeOperatorStateHandle(operatorStateHandle, dos); } } else { // this means no states, not even an empty list @@ -83,7 +83,7 @@ public static void writeStateHandle(OperatorSubtaskState state, String path) thr if (rawKeyedState != null) { dos.writeInt(rawKeyedState.size()); for (KeyedStateHandle keyedStateHandle : rawKeyedState) { - SavepointV1Serializer.serializeKeyedStateHandle(keyedStateHandle, dos); + SavepointV2Serializer.serializeKeyedStateHandle(keyedStateHandle, dos); } } else { // this means no operator states, not even an empty list @@ -94,7 +94,7 @@ public static void writeStateHandle(OperatorSubtaskState state, String path) thr if (managedKeyedState != null) { dos.writeInt(managedKeyedState.size()); for (KeyedStateHandle keyedStateHandle : managedKeyedState) { - SavepointV1Serializer.serializeKeyedStateHandle(keyedStateHandle, dos); + SavepointV2Serializer.serializeKeyedStateHandle(keyedStateHandle, dos); } } else { // this means no operator states, not even an empty list @@ -113,14 +113,14 @@ public static OperatorSubtaskState readStateHandle(String path) throws IOExcepti dis.readInt(); // still required for compatibility to consume the bytes. - SavepointV1Serializer.deserializeStreamStateHandle(dis); + SavepointV2Serializer.deserializeStreamStateHandle(dis); List rawOperatorState = null; int numRawOperatorStates = dis.readInt(); if (numRawOperatorStates >= 0) { rawOperatorState = new ArrayList<>(); for (int i = 0; i < numRawOperatorStates; i++) { - OperatorStateHandle operatorState = SavepointV1Serializer.deserializeOperatorStateHandle( + OperatorStateHandle operatorState = SavepointV2Serializer.deserializeOperatorStateHandle( dis); rawOperatorState.add(operatorState); } @@ -131,7 +131,7 @@ public static OperatorSubtaskState readStateHandle(String path) throws IOExcepti if (numManagedOperatorStates >= 0) { managedOperatorState = new ArrayList<>(); for (int i = 0; i < numManagedOperatorStates; i++) { - OperatorStateHandle operatorState = SavepointV1Serializer.deserializeOperatorStateHandle( + OperatorStateHandle operatorState = SavepointV2Serializer.deserializeOperatorStateHandle( dis); managedOperatorState.add(operatorState); } @@ -142,7 +142,7 @@ public static OperatorSubtaskState readStateHandle(String path) throws IOExcepti if (numRawKeyedStates >= 0) { rawKeyedState = new ArrayList<>(); for (int i = 0; i < numRawKeyedStates; i++) { - KeyedStateHandle keyedState = SavepointV1Serializer.deserializeKeyedStateHandle( + KeyedStateHandle keyedState = SavepointV2Serializer.deserializeKeyedStateHandle( dis); rawKeyedState.add(keyedState); } @@ -153,7 +153,7 @@ public static OperatorSubtaskState readStateHandle(String path) throws IOExcepti if (numManagedKeyedStates >= 0) { managedKeyedState = new ArrayList<>(); for (int i = 0; i < numManagedKeyedStates; i++) { - KeyedStateHandle keyedState = SavepointV1Serializer.deserializeKeyedStateHandle( + KeyedStateHandle keyedState = SavepointV2Serializer.deserializeKeyedStateHandle( dis); managedKeyedState.add(keyedState); } diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala index 23d216549f836..15dca2c9c0752 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala @@ -915,13 +915,19 @@ class DataStream[T](stream: JavaStream[T]) { * Operator used for directing tuples to specific named outputs using an * OutputSelector. Calling this method on an operator creates a new * [[SplitStream]]. + * + * @deprecated Please use side output instead. */ + @deprecated def split(selector: OutputSelector[T]): SplitStream[T] = asScalaStream(stream.split(selector)) /** * Creates a new [[SplitStream]] that contains only the elements satisfying the * given output selector predicate. + * + * @deprecated Please use side output instead. */ + @deprecated def split(fun: T => TraversableOnce[String]): SplitStream[T] = { if (fun == null) { throw new NullPointerException("OutputSelector must not be null.") diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala index 580dd46d68342..f85e1b538e2c8 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala @@ -201,7 +201,12 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T] * @return Returns a DataStream */ @PublicEvolving - def process[OUT](processJoinFunction: ProcessJoinFunction[IN1, IN2, OUT]): DataStream[OUT] = { + def process[OUT: TypeInformation]( + processJoinFunction: ProcessJoinFunction[IN1, IN2, OUT]) + : DataStream[OUT] = { + + val outType: TypeInformation[OUT] = implicitly[TypeInformation[OUT]] + val javaJoined = new KeyedJavaStream.IntervalJoined[IN1, IN2, KEY]( firstStream.javaStream.asInstanceOf[KeyedJavaStream[IN1, KEY]], secondStream.javaStream.asInstanceOf[KeyedJavaStream[IN2, KEY]], @@ -209,7 +214,7 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T] upperBound, lowerBoundInclusive, upperBoundInclusive) - asScalaStream(javaJoined.process(processJoinFunction)) + asScalaStream(javaJoined.process(processJoinFunction, outType)) } } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala index 80701c6725a8c..a9ace36cc4960 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala @@ -18,6 +18,7 @@ package org.apache.flink.streaming.api.scala +import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction import org.apache.flink.streaming.api.functions.sink.SinkFunction @@ -25,7 +26,8 @@ import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExt import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.util.Collector -import org.junit.{Assert, Test} +import org.junit.Assert.assertTrue +import org.junit.Test import scala.collection.mutable.ListBuffer @@ -47,22 +49,25 @@ class IntervalJoinITCase extends AbstractTestBase { val sink = new ResultSink() - dataStream1.intervalJoin(dataStream2) + val join = dataStream1.intervalJoin(dataStream2) .between(Time.milliseconds(0), Time.milliseconds(2)) - .process(new CombineToStringJoinFunction()) - .addSink(sink) + .process(new CombineJoinFunction()) + + assertTrue(join.dataType.isInstanceOf[CaseClassTypeInfo[_]]) + + join.addSink(sink) env.execute() sink.expectInAnyOrder( - "(key,0):(key,0)", - "(key,0):(key,1)", - "(key,0):(key,2)", + "(key:key,0)", + "(key:key,1)", + "(key:key,2)", - "(key,1):(key,1)", - "(key,1):(key,2)", + "(key:key,2)", + "(key:key,3)", - "(key,2):(key,2)" + "(key:key,4)" ) } @@ -82,18 +87,21 @@ class IntervalJoinITCase extends AbstractTestBase { val sink = new ResultSink() - dataStream1.intervalJoin(dataStream2) + val join = dataStream1.intervalJoin(dataStream2) .between(Time.milliseconds(0), Time.milliseconds(2)) .lowerBoundExclusive() .upperBoundExclusive() - .process(new CombineToStringJoinFunction()) - .addSink(sink) + .process(new CombineJoinFunction()) + + assertTrue(join.dataType.isInstanceOf[CaseClassTypeInfo[_]]) + + join.addSink(sink) env.execute() sink.expectInAnyOrder( - "(key,0):(key,1)", - "(key,1):(key,2)" + "(key:key,1)", + "(key:key,3)" ) } } @@ -102,14 +110,14 @@ object Companion { val results: ListBuffer[String] = new ListBuffer() } -class ResultSink extends SinkFunction[String] { +class ResultSink extends SinkFunction[(String, Long)] { - override def invoke(value: String, context: SinkFunction.Context[_]): Unit = { - Companion.results.append(value) + override def invoke(value: (String, Long), context: SinkFunction.Context[_]): Unit = { + Companion.results.append(value.toString()) } def expectInAnyOrder(expected: String*): Unit = { - Assert.assertTrue(expected.toSet.equals(Companion.results.toSet)) + assertTrue(expected.toSet.equals(Companion.results.toSet)) } } @@ -117,14 +125,14 @@ class TimestampExtractor extends AscendingTimestampExtractor[(String, Long)] { override def extractAscendingTimestamp(element: (String, Long)): Long = element._2 } -class CombineToStringJoinFunction - extends ProcessJoinFunction[(String, Long), (String, Long), String] { +class CombineJoinFunction + extends ProcessJoinFunction[(String, Long), (String, Long), (String, Long)] { override def processElement( - left: (String, Long), - right: (String, Long), - ctx: ProcessJoinFunction[(String, Long), (String, Long), String]#Context, - out: Collector[String]): Unit = { - out.collect(left + ":" + right) + left: (String, Long), + right: (String, Long), + ctx: ProcessJoinFunction[(String, Long), (String, Long), (String, Long)]#Context, + out: Collector[(String, Long)]): Unit = { + out.collect((left._1 + ":" + right._1, left._2 + right._2)) } } diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/s3/S3TestCredentials.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/s3/S3TestCredentials.java new file mode 100644 index 0000000000000..b1a0aecf07971 --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/s3/S3TestCredentials.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.testutils.s3; + +import org.junit.Assume; +import org.junit.AssumptionViolatedException; + +import javax.annotation.Nullable; + +/** + * Access to credentials to access S3 buckets during integration tests. + */ +public class S3TestCredentials { + + @Nullable + private static final String S3_TEST_BUCKET = System.getenv("IT_CASE_S3_BUCKET"); + + @Nullable + private static final String S3_TEST_ACCESS_KEY = System.getenv("IT_CASE_S3_ACCESS_KEY"); + + @Nullable + private static final String S3_TEST_SECRET_KEY = System.getenv("IT_CASE_S3_SECRET_KEY"); + + // ------------------------------------------------------------------------ + + /** + * Checks whether S3 test credentials are available in the environment variables + * of this JVM. + */ + public static boolean credentialsAvailable() { + return S3_TEST_BUCKET != null && S3_TEST_ACCESS_KEY != null && S3_TEST_SECRET_KEY != null; + } + + /** + * Checks whether credentials are available in the environment variables of this JVM. + * If not, throws an {@link AssumptionViolatedException} which causes JUnit tests to be + * skipped. + */ + public static void assumeCredentialsAvailable() { + Assume.assumeTrue("No S3 credentials available in this test's environment", credentialsAvailable()); + } + + /** + * Gets the S3 Access Key. + * + *

    This method throws an exception if the key is not available. Tests should + * use {@link #assumeCredentialsAvailable()} to skip tests when credentials are not + * available. + */ + public static String getS3AccessKey() { + if (S3_TEST_ACCESS_KEY != null) { + return S3_TEST_ACCESS_KEY; + } + else { + throw new IllegalStateException("S3 test access key not available"); + } + } + + /** + * Gets the S3 Secret Key. + * + *

    This method throws an exception if the key is not available. Tests should + * use {@link #assumeCredentialsAvailable()} to skip tests when credentials are not + * available. + */ + public static String getS3SecretKey() { + if (S3_TEST_SECRET_KEY != null) { + return S3_TEST_SECRET_KEY; + } + else { + throw new IllegalStateException("S3 test secret key not available"); + } + } + + /** + * Gets the URI for the path under which all tests should put their data. + * + *

    This method throws an exception if the bucket was not configured. Tests should + * use {@link #assumeCredentialsAvailable()} to skip tests when credentials are not + * available. + */ + public static String getTestBucketUri() { + return getTestBucketUriWithScheme("s3"); + } + + /** + * Gets the URI for the path under which all tests should put their data. + * + *

    This method throws an exception if the bucket was not configured. Tests should + * use {@link #assumeCredentialsAvailable()} to skip tests when credentials are not + * available. + */ + public static String getTestBucketUriWithScheme(String scheme) { + if (S3_TEST_BUCKET != null) { + return scheme + "://" + S3_TEST_BUCKET + "/temp/"; + } + else { + throw new IllegalStateException("S3 test bucket not available"); + } + } +} diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml index 6c32a436babc2..d1c829fe82d3e 100644 --- a/flink-test-utils-parent/flink-test-utils/pom.xml +++ b/flink-test-utils-parent/flink-test-utils/pom.xml @@ -143,11 +143,17 @@ under the License. org.apache.flink.shaded.testutils.org.jboss.netty - + - * + io.netty:netty META-INF/maven/io.netty/** + + META-INF/license + + META-INF/NOTICE.txt diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/testutils/FiniteTestSource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java similarity index 72% rename from flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/testutils/FiniteTestSource.java rename to flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java index 03db7ff4e8161..b3a9546b3ddeb 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/testutils/FiniteTestSource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.formats.parquet.testutils; +package org.apache.flink.streaming.util; import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.functions.source.SourceFunction; @@ -24,8 +24,15 @@ import java.util.Arrays; /** - * A stream source that emits elements without allowing checkpoints and waits - * for two more checkpoints to complete before exiting. + * A stream source that: + * 1) emits a list of elements without allowing checkpoints, + * 2) then waits for two more checkpoints to complete, + * 3) then re-emits the same elements before + * 4) waiting for another two checkpoints and + * 5) exiting. + * + *

    This class was written to test the Bulk Writers used by + * the StreamingFileSink. */ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") public class FiniteTestSource implements SourceFunction, CheckpointListener { @@ -50,11 +57,19 @@ public FiniteTestSource(Iterable elements) { @Override public void run(SourceContext ctx) throws Exception { + // first round of sending the elements and waiting for the checkpoints + emitElementsAndWaitForCheckpoints(ctx, 2); + + // second round of the same + emitElementsAndWaitForCheckpoints(ctx, 2); + } + + private void emitElementsAndWaitForCheckpoints(SourceContext ctx, int checkpointsToWaitFor) throws InterruptedException { final Object lock = ctx.getCheckpointLock(); - final int checkpointToAwait; + final int checkpointToAwait; synchronized (lock) { - checkpointToAwait = numCheckpointsComplete + 2; + checkpointToAwait = numCheckpointsComplete + checkpointsToWaitFor; for (T t : elements) { ctx.collect(t); } diff --git a/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/NOTICE b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000..07a9b6bb4283f --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/NOTICE @@ -0,0 +1,40 @@ +flink-test-utils +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.netty:netty:3.10.6.Final + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit diff --git a/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 0000000000000..31ebc840539c1 --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 0000000000000..b1c292b54cb2b --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 0000000000000..29ad562af0a5e --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 0000000000000..ec5f348998bd4 --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java index 0ead861adbaf7..d4063184b62d4 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java @@ -153,7 +153,7 @@ public void go() throws Exception { try { NotifyingMapper.notifyLatch.await(); - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> { try { return CompletableFuture.completedFuture(client.getAccumulators(jobGraph.getJobID())); diff --git a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancelingTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancelingTestBase.java index 0d29568c2a045..142ca43dda44e 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancelingTestBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancelingTestBase.java @@ -53,6 +53,8 @@ public abstract class CancelingTestBase extends TestLogger { protected static final int PARALLELISM = 4; + protected static final long GET_FUTURE_TIMEOUT = 1000; // 1000 milliseconds + // -------------------------------------------------------------------------------------------- @ClassRule @@ -95,10 +97,10 @@ protected void runAndCancelJob(Plan plan, final int msecsTillCanceling, int maxT Deadline submissionDeadLine = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); - JobStatus jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(submissionDeadLine.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + JobStatus jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); while (jobStatus != JobStatus.RUNNING && submissionDeadLine.hasTimeLeft()) { Thread.sleep(50); - jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(submissionDeadLine.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); } if (jobStatus != JobStatus.RUNNING) { Assert.fail("Job not in state RUNNING."); @@ -110,10 +112,10 @@ protected void runAndCancelJob(Plan plan, final int msecsTillCanceling, int maxT Deadline cancelDeadline = new FiniteDuration(maxTimeTillCanceled, TimeUnit.MILLISECONDS).fromNow(); - JobStatus jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(cancelDeadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + JobStatus jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); while (jobStatusAfterCancel != JobStatus.CANCELED && cancelDeadline.hasTimeLeft()) { Thread.sleep(50); - jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(cancelDeadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); } if (jobStatusAfterCancel != JobStatus.CANCELED) { Assert.fail("Failed to cancel job with ID " + jobSubmissionResult.getJobID() + '.'); diff --git a/flink-tests/src/test/java/org/apache/flink/test/cancelling/JoinCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/cancelling/JoinCancelingITCase.java index 7288ba53762bd..9a6ae44b52b09 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/cancelling/JoinCancelingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/cancelling/JoinCancelingITCase.java @@ -20,19 +20,21 @@ import org.apache.flink.api.common.functions.JoinFunction; import org.apache.flink.api.common.functions.RichJoinFunction; +import org.apache.flink.api.common.io.GenericInputFormat; import org.apache.flink.api.common.operators.base.JoinOperatorBase; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.io.DiscardingOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator; import org.apache.flink.test.util.InfiniteIntegerTupleInputFormat; import org.apache.flink.test.util.UniformIntTupleGeneratorInputFormat; import org.junit.Ignore; import org.junit.Test; +import java.io.IOException; + /** * Test job cancellation from within a JoinFunction. */ @@ -78,7 +80,6 @@ public void testCancelSortMatchPriorToFirstRecordReading() throws Exception { private void executeTaskWithGenerator( JoinFunction, Tuple2, Tuple2> joiner, int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception { - UniformIntTupleGenerator g = new UniformIntTupleGenerator(keys, vals, false); ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals)); DataSet> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals)); @@ -96,7 +97,25 @@ private void executeTaskWithGenerator( @Test public void testCancelSortMatchWhileDoingHeavySorting() throws Exception { - executeTaskWithGenerator(new SimpleMatcher(), 50000, 100, 30 * 1000, 30 * 1000); + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + HeavyCompareGeneratorInputFormat input = new HeavyCompareGeneratorInputFormat(100); + DataSet> input1 = env.createInput(input); + DataSet> input2 = env.createInput(input); + + input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE) + .where(0) + .equalTo(0) + .with(new JoinFunction, Tuple2, Tuple2>() { + @Override + public Tuple2 join( + Tuple2 first, + Tuple2 second) throws Exception { + throw new Exception("Job should be canceled in sort-merge phase, never run here ..."); + } + }) + .output(new DiscardingOutputFormat>()); + + runAndCancelJob(env.createProgramPlan(), 30 * 1000, 60 * 1000); } // --------------- Test Sort Matches that are canceled while in the Matching Phase ----------------- @@ -175,3 +194,32 @@ public Tuple2 join(Tuple2 first, Tuple2 second) throws E } } } + +class HeavyCompare implements Comparable, java.io.Serializable { + @Override + public int compareTo(org.apache.flink.test.cancelling.HeavyCompare o) { + try { + Thread.sleep(1000); + } catch (InterruptedException iex) {} + return 0; + } +} + +class HeavyCompareGeneratorInputFormat extends GenericInputFormat> { + private int valueTotal; + + public HeavyCompareGeneratorInputFormat(int numVals) { + this.valueTotal = numVals; + } + + @Override + public boolean reachedEnd() throws IOException { + return valueTotal <= 0; + } + + @Override + public Tuple2 nextRecord(Tuple2 reuse) throws IOException { + valueTotal -= 1; + return new Tuple2<>(new HeavyCompare(), 20110701); + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java index 53747659d320f..a8626f34e33d6 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java @@ -87,6 +87,7 @@ protected Configuration createClusterConfig() throws IOException { private void executeTest(EventTimeWindowCheckpointingITCase delegate) throws Exception { delegate.name = testName; + delegate.stateBackendEnum = backendEnum; try { delegate.setupTestCluster(); try { @@ -94,6 +95,7 @@ private void executeTest(EventTimeWindowCheckpointingITCase delegate) throws Exc delegate.stopTestCluster(); } catch (Exception e) { delegate.stopTestCluster(); + throw new RuntimeException(e); } delegate.setupTestCluster(); @@ -102,9 +104,10 @@ private void executeTest(EventTimeWindowCheckpointingITCase delegate) throws Exc delegate.stopTestCluster(); } catch (Exception e) { delegate.stopTestCluster(); + throw new RuntimeException(e); } } finally { - delegate.tempFolder.delete(); + EventTimeWindowCheckpointingITCase.tempFolder.delete(); } } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java index 2cd2bbb60e926..67cc9b7aec9e6 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java @@ -53,11 +53,16 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.test.util.MiniClusterWithClientResource; +import org.apache.flink.testutils.EntropyInjectingTestFileSystem; import org.apache.flink.util.Collector; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; import org.junit.Assert; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -66,7 +71,12 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.Collections; @@ -76,9 +86,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -93,6 +106,22 @@ public class SavepointITCase extends TestLogger { @Rule public final TemporaryFolder folder = new TemporaryFolder(); + private File checkpointDir; + + private File savepointDir; + + @Before + public void setUp() throws Exception { + final File testRoot = folder.newFolder(); + + checkpointDir = new File(testRoot, "checkpoints"); + savepointDir = new File(testRoot, "savepoints"); + + if (!checkpointDir.mkdir() || !savepointDir.mkdirs()) { + fail("Test setup failed: failed to create temporary directories."); + } + } + /** * Triggers a savepoint for a job that uses the FsStateBackend. We expect * that all checkpoint files are written to a new savepoint directory. @@ -109,35 +138,46 @@ public class SavepointITCase extends TestLogger { */ @Test public void testTriggerSavepointAndResumeWithFileBasedCheckpoints() throws Exception { - // Config final int numTaskManagers = 2; final int numSlotsPerTaskManager = 2; final int parallelism = numTaskManagers * numSlotsPerTaskManager; - final File testRoot = folder.newFolder(); - Configuration config = new Configuration(); + final MiniClusterResourceFactory clusterFactory = new MiniClusterResourceFactory( + numTaskManagers, + numSlotsPerTaskManager, + getFileBasedCheckpointsConfig()); - final File checkpointDir = new File(testRoot, "checkpoints"); - final File savepointRootDir = new File(testRoot, "savepoints"); + final String savepointPath = submitJobAndTakeSavepoint(clusterFactory, parallelism); + verifySavepoint(parallelism, savepointPath); - if (!checkpointDir.mkdir() || !savepointRootDir.mkdirs()) { - fail("Test setup failed: failed to create temporary directories."); - } + restoreJobAndVerifyState(savepointPath, clusterFactory, parallelism); + } - // Use file based checkpoints - config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem"); - config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); - config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0); - config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointRootDir.toURI().toString()); + @Test + public void testShouldAddEntropyToSavepointPath() throws Exception { + final int numTaskManagers = 2; + final int numSlotsPerTaskManager = 2; + final int parallelism = numTaskManagers * numSlotsPerTaskManager; - MiniClusterResourceFactory clusterFactory = new MiniClusterResourceFactory(numTaskManagers, numSlotsPerTaskManager, config); + final MiniClusterResourceFactory clusterFactory = new MiniClusterResourceFactory( + numTaskManagers, + numSlotsPerTaskManager, + getCheckpointingWithEntropyConfig()); - String savepointPath = submitJobAndGetVerifiedSavepoint(clusterFactory, parallelism); + final String savepointPath = submitJobAndTakeSavepoint(clusterFactory, parallelism); + assertThat(savepointDir, hasEntropyInFileStateHandlePaths()); restoreJobAndVerifyState(savepointPath, clusterFactory, parallelism); } - private String submitJobAndGetVerifiedSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { + private Configuration getCheckpointingWithEntropyConfig() { + final String savepointPathWithEntropyPlaceholder = new File(savepointDir, EntropyInjectingTestFileSystem.ENTROPY_INJECTION_KEY).getPath(); + final Configuration config = getFileBasedCheckpointsConfig("test-entropy://" + savepointPathWithEntropyPlaceholder); + config.setString("s3.entropy.key", EntropyInjectingTestFileSystem.ENTROPY_INJECTION_KEY); + return config; + } + + private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); final JobID jobId = jobGraph.getJobID(); StatefulCounter.resetForTest(parallelism); @@ -152,32 +192,32 @@ private String submitJobAndGetVerifiedSavepoint(MiniClusterResourceFactory clust StatefulCounter.getProgressLatch().await(); - String savepointPath = client.triggerSavepoint(jobId, null).get(); - - // Only one savepoint should exist - File savepointDir = new File(new URI(savepointPath)); - assertTrue("Savepoint directory does not exist.", savepointDir.exists()); - assertTrue("Savepoint did not create self-contained directory.", savepointDir.isDirectory()); - - File[] savepointFiles = savepointDir.listFiles(); - - if (savepointFiles != null) { - // Expect one metadata file and one checkpoint file per stateful - // parallel subtask - String errMsg = "Did not write expected number of savepoint/checkpoint files to directory: " - + Arrays.toString(savepointFiles); - assertEquals(errMsg, 1 + parallelism, savepointFiles.length); - } else { - fail(String.format("Returned savepoint path (%s) is not valid.", savepointPath)); - } - - return savepointPath; + return client.cancelWithSavepoint(jobId, null); } finally { cluster.after(); StatefulCounter.resetForTest(parallelism); } } + private void verifySavepoint(final int parallelism, final String savepointPath) throws URISyntaxException { + // Only one savepoint should exist + File savepointDir = new File(new URI(savepointPath)); + assertTrue("Savepoint directory does not exist.", savepointDir.exists()); + assertTrue("Savepoint did not create self-contained directory.", savepointDir.isDirectory()); + + File[] savepointFiles = savepointDir.listFiles(); + + if (savepointFiles != null) { + // Expect one metadata file and one checkpoint file per stateful + // parallel subtask + String errMsg = "Did not write expected number of savepoint/checkpoint files to directory: " + + Arrays.toString(savepointFiles); + assertEquals(errMsg, 1 + parallelism, savepointFiles.length); + } else { + fail(String.format("Returned savepoint path (%s) is not valid.", savepointPath)); + } + } + private void restoreJobAndVerifyState(String savepointPath, MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); @@ -200,7 +240,7 @@ private void restoreJobAndVerifyState(String savepointPath, MiniClusterResourceF client.cancel(jobId); - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> client.getJobStatus(jobId), Time.milliseconds(50), Deadline.now().plus(Duration.ofSeconds(30)), @@ -224,9 +264,6 @@ public void testTriggerSavepointForNonExistingJob() throws Exception { final int numTaskManagers = 1; final int numSlotsPerTaskManager = 1; - final File tmpDir = folder.newFolder(); - final File savepointDir = new File(tmpDir, "savepoints"); - final Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); @@ -299,9 +336,6 @@ public void testSubmitWithUnknownSavepointPath() throws Exception { int numSlotsPerTaskManager = 1; int parallelism = numTaskManagers * numSlotsPerTaskManager; - final File tmpDir = folder.newFolder(); - final File savepointDir = new File(tmpDir, "savepoints"); - final Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); @@ -361,9 +395,6 @@ public void testCanRestoreWithModifiedStatelessOperators() throws Exception { // Test deadline final Deadline deadline = Deadline.now().plus(Duration.ofMinutes(5)); - final File tmpDir = folder.newFolder(); - final File savepointDir = new File(tmpDir, "savepoints"); - // Flink configuration final Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); @@ -590,11 +621,6 @@ public void testSavepointForJobWithIteration() throws Exception { iterTestCheckpointVerify[i] = 0; } - TemporaryFolder folder = new TemporaryFolder(); - folder.create(); - // Temporary directory for file state backend - final File tmpDir = folder.newFolder(); - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); final IntegerStreamSource source = new IntegerStreamSource(); IterativeStream iteration = env.addSource(source) @@ -638,20 +664,9 @@ public Integer map(Integer value) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); - Configuration config = new Configuration(); + Configuration config = getFileBasedCheckpointsConfig(); config.addAll(jobGraph.getJobConfiguration()); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); - final File checkpointDir = new File(tmpDir, "checkpoints"); - final File savepointDir = new File(tmpDir, "savepoints"); - - if (!checkpointDir.mkdir() || !savepointDir.mkdirs()) { - fail("Test setup failed: failed to create temporary directories."); - } - - config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem"); - config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); - config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0); - config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() @@ -798,4 +813,65 @@ MiniClusterWithClientResource get() { .build()); } } + + private Configuration getFileBasedCheckpointsConfig(final String savepointDir) { + final Configuration config = new Configuration(); + config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem"); + config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); + config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0); + config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); + return config; + } + + private Configuration getFileBasedCheckpointsConfig() { + return getFileBasedCheckpointsConfig(savepointDir.toURI().toString()); + } + + private static Matcher hasEntropyInFileStateHandlePaths() { + return new TypeSafeDiagnosingMatcher() { + + @Override + protected boolean matchesSafely(final File savepointDir, final Description mismatchDescription) { + if (savepointDir == null) { + mismatchDescription.appendText("savepoint dir must not be null"); + return false; + } + + final List filesWithoutEntropy = listRecursively(savepointDir.toPath().resolve(EntropyInjectingTestFileSystem.ENTROPY_INJECTION_KEY)); + final Path savepointDirWithEntropy = savepointDir.toPath().resolve(EntropyInjectingTestFileSystem.ENTROPY); + final List filesWithEntropy = listRecursively(savepointDirWithEntropy); + + if (!filesWithoutEntropy.isEmpty()) { + mismatchDescription.appendText("there are savepoint files with unresolved entropy placeholders"); + return false; + } + + if (!Files.exists(savepointDirWithEntropy) || filesWithEntropy.isEmpty()) { + mismatchDescription.appendText("there are no savepoint files with added entropy"); + return false; + } + + return true; + } + + @Override + public void describeTo(final Description description) { + description.appendText("all savepoint files should have added entropy"); + } + }; + } + + private static List listRecursively(final Path dir) { + try { + if (!Files.exists(dir)) { + return Collections.emptyList(); + } else { + try (Stream files = Files.walk(dir, FileVisitOption.FOLLOW_LINKS)) { + return files.filter(Files::isRegularFile).collect(Collectors.toList()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java index 7c00de7f43684..9bec331608ea6 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java @@ -258,7 +258,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO // now the job should be able to go to RUNNING again and then eventually to FINISHED, // which it only does if it could successfully restore - CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + CompletableFuture jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobID), Time.milliseconds(50), deadline, diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java index f792362085050..3f49d84d59594 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java @@ -113,17 +113,17 @@ public void testSavepoint() throws Exception { expectedFirstState.put(2L, 2L); expectedFirstState.put(3L, 3L); - final Map expectedSecondState = new HashMap<>(); - expectedSecondState.put("0", "0"); - expectedSecondState.put("1", "1"); - expectedSecondState.put("2", "2"); - expectedSecondState.put("3", "3"); - - final Map expectedThirdState = new HashMap<>(); - expectedThirdState.put("0", "0"); - expectedThirdState.put("1", "1"); - expectedThirdState.put("2", "2"); - expectedThirdState.put("3", "3"); + final Map expectedSecondState = new HashMap<>(); + expectedSecondState.put("0", 0L); + expectedSecondState.put("1", 1L); + expectedSecondState.put("2", 2L); + expectedSecondState.put("3", 3L); + + final Map expectedThirdState = new HashMap<>(); + expectedThirdState.put(0L, "0"); + expectedThirdState.put(1L, "1"); + expectedThirdState.put(2L, "2"); + expectedThirdState.put(3L, "3"); if (executionMode == StatefulJobSavepointMigrationITCase.ExecutionMode.PERFORM_SAVEPOINT) { nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS); @@ -171,12 +171,12 @@ public Long getKey(Tuple2 value) throws Exception { "broadcast-state-1", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); - final MapStateDescriptor secondBroadcastStateDesc = new MapStateDescriptor<>( - "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + final MapStateDescriptor secondBroadcastStateDesc = new MapStateDescriptor<>( + "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); - final MapStateDescriptor thirdBroadcastStateDesc = new MapStateDescriptor<>( - "broadcast-state-3", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + final MapStateDescriptor thirdBroadcastStateDesc = new MapStateDescriptor<>( + "broadcast-state-3", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO ); BroadcastStream> npBroadcastStream = env @@ -234,7 +234,7 @@ private static class CheckpointingKeyedBroadcastFunction private MapStateDescriptor firstStateDesc; - private MapStateDescriptor secondStateDesc; + private MapStateDescriptor secondStateDesc; @Override public void open(Configuration parameters) throws Exception { @@ -245,7 +245,7 @@ public void open(Configuration parameters) throws Exception { ); secondStateDesc = new MapStateDescriptor<>( - "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); } @@ -257,7 +257,7 @@ public void processElement(Tuple2 value, ReadOnlyContext ctx, Collec @Override public void processBroadcastElement(Tuple2 value, Context ctx, Collector> out) throws Exception { ctx.getBroadcastState(firstStateDesc).put(value.f0, value.f1); - ctx.getBroadcastState(secondStateDesc).put(Long.toString(value.f0), Long.toString(value.f1)); + ctx.getBroadcastState(secondStateDesc).put(Long.toString(value.f0), value.f1); } } @@ -269,14 +269,14 @@ private static class CheckpointingKeyedSingleBroadcastFunction private static final long serialVersionUID = 1333992081671604521L; - private MapStateDescriptor stateDesc; + private MapStateDescriptor stateDesc; @Override public void open(Configuration parameters) throws Exception { super.open(parameters); stateDesc = new MapStateDescriptor<>( - "broadcast-state-3", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + "broadcast-state-3", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO ); } @@ -287,7 +287,7 @@ public void processElement(Tuple2 value, ReadOnlyContext ctx, Collec @Override public void processBroadcastElement(Tuple2 value, Context ctx, Collector> out) throws Exception { - ctx.getBroadcastState(stateDesc).put(Long.toString(value.f0), Long.toString(value.f1)); + ctx.getBroadcastState(stateDesc).put(value.f0, Long.toString(value.f1)); } } @@ -301,13 +301,13 @@ private static class CheckingKeyedBroadcastFunction private final Map expectedFirstState; - private final Map expectedSecondState; + private final Map expectedSecondState; private MapStateDescriptor firstStateDesc; - private MapStateDescriptor secondStateDesc; + private MapStateDescriptor secondStateDesc; - CheckingKeyedBroadcastFunction(Map firstState, Map secondState) { + CheckingKeyedBroadcastFunction(Map firstState, Map secondState) { this.expectedFirstState = firstState; this.expectedSecondState = secondState; } @@ -321,7 +321,7 @@ public void open(Configuration parameters) throws Exception { ); secondStateDesc = new MapStateDescriptor<>( - "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + "broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); } @@ -334,8 +334,8 @@ public void processElement(Tuple2 value, ReadOnlyContext ctx, Collec } Assert.assertEquals(expectedFirstState, actualFirstState); - final Map actualSecondState = new HashMap<>(); - for (Map.Entry entry: ctx.getBroadcastState(secondStateDesc).immutableEntries()) { + final Map actualSecondState = new HashMap<>(); + for (Map.Entry entry: ctx.getBroadcastState(secondStateDesc).immutableEntries()) { actualSecondState.put(entry.getKey(), entry.getValue()); } Assert.assertEquals(expectedSecondState, actualSecondState); @@ -357,11 +357,11 @@ private static class CheckingKeyedSingleBroadcastFunction private static final long serialVersionUID = 1333992081671604521L; - private final Map expectedState; + private final Map expectedState; - private MapStateDescriptor stateDesc; + private MapStateDescriptor stateDesc; - CheckingKeyedSingleBroadcastFunction(Map state) { + CheckingKeyedSingleBroadcastFunction(Map state) { this.expectedState = state; } @@ -370,14 +370,14 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); stateDesc = new MapStateDescriptor<>( - "broadcast-state-3", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO + "broadcast-state-3", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO ); } @Override public void processElement(Tuple2 value, ReadOnlyContext ctx, Collector> out) throws Exception { - final Map actualState = new HashMap<>(); - for (Map.Entry entry: ctx.getBroadcastState(stateDesc).immutableEntries()) { + final Map actualState = new HashMap<>(); + for (Map.Entry entry: ctx.getBroadcastState(stateDesc).immutableEntries()) { actualState.put(entry.getKey(), entry.getValue()); } Assert.assertEquals(expectedState, actualState); diff --git a/flink-tests/src/test/java/org/apache/flink/test/planning/LargePlanTest.java b/flink-tests/src/test/java/org/apache/flink/test/planning/LargePlanTest.java new file mode 100644 index 0000000000000..9b2dbd122724c --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/planning/LargePlanTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.planning; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.api.java.io.DiscardingOutputFormat; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.client.program.OptimizerPlanEnvironment; +import org.apache.flink.client.program.PreviewPlanEnvironment; + +import org.junit.Test; + +/** + * Tests that large programs can be compiled to a Plan in reasonable amount of time. + */ +public class LargePlanTest { + + @Test(expected = OptimizerPlanEnvironment.ProgramAbortException.class, timeout = 30_000) + public void testPlanningOfLargePlan() throws Exception { + runProgram(new PreviewPlanEnvironment(), 10, 20); + } + + private static void runProgram(ExecutionEnvironment env, int depth, int width) throws Exception { + DataSet input = env.fromElements("a", "b", "c"); + DataSet stats = null; + + for (int i = 0; i < depth; i++) { + stats = analyze(input, stats, width / (i + 1) + 1); + } + + stats.output(new DiscardingOutputFormat<>()); + env.execute("depth " + depth + " width " + width); + } + + private static DataSet analyze(DataSet input, DataSet stats, int branches) { + for (int i = 0; i < branches; i++) { + final int ii = i; + + if (stats != null) { + input = input.map( + new RichMapFunction() { + @Override + public String map(String value) { + return value; + } + }).withBroadcastSet(stats.map(s -> "(" + s + ").map"), "stats"); + } + + DataSet branch = input + .map(s -> new Tuple2<>(0, s + ii)).returns(Types.TUPLE(Types.STRING, Types.INT)) + .groupBy(0) + .minBy(1) + .map(kv -> kv.f1).returns(Types.STRING); + if (stats == null) { + stats = branch; + } else { + stats = stats.union(branch); + } + } + return stats.map(s -> "(" + s + ").stats"); + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java index ec3f1e1b9d3fa..e83f9ab8c583f 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java @@ -393,7 +393,7 @@ public void run() { } private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException { - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())), Time.milliseconds(50L), org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())), diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java index 90ea8796ca57c..96e289dbf5ccd 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java @@ -32,6 +32,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.HighAvailabilityOptions; import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.clusterframework.ApplicationStatus; @@ -116,6 +117,7 @@ public void testCancelingOnProcessFailure() throws Exception { config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 100); + config.setInteger(RestOptions.PORT, 0); final RpcService rpcService = AkkaRpcServiceUtils.createRpcService("localhost", 0, config); final int jobManagerPort = rpcService.getPort(); @@ -294,7 +296,7 @@ static DispatcherGateway retrieveDispatcherGateway(RpcService rpcService, HighAv } private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException { - FutureUtils.retrySuccesfulWithDelay( + FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(timeout), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), @@ -306,7 +308,7 @@ private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time } private Collection waitForRunningJobs(ClusterClient clusterClient, Time timeout) throws ExecutionException, InterruptedException { - return FutureUtils.retrySuccesfulWithDelay( + return FutureUtils.retrySuccessfulWithDelay( CheckedSupplier.unchecked(clusterClient::listJobs), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java index 6783df0a85b0a..1a5b2a32f9b75 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java @@ -33,6 +33,7 @@ public class SimpleRecoveryFailureRateStrategyITBase extends SimpleRecoveryITCas @ClassRule public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() + .setConfiguration(getConfiguration()) .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(2) .build()); diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java index d81274d497341..43de1f4645b30 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java @@ -110,6 +110,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper1.failuresBeforeSuccess = 1; } } @@ -158,6 +160,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper2.failuresBeforeSuccess = 1; } } @@ -194,6 +198,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper3.failuresBeforeSuccess = 3; } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java index 3db0f62f8292a..0b79af513f6bb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java @@ -118,7 +118,7 @@ private String migrateJob(ClassLoader classLoader, ClusterClient clusterClien clusterClient.submitJob(jobToMigrate, classLoader); - CompletableFuture jobRunningFuture = FutureUtils.retrySuccesfulWithDelay( + CompletableFuture jobRunningFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobToMigrate.getJobID()), Time.milliseconds(50), deadline, @@ -142,6 +142,7 @@ private String migrateJob(ClassLoader classLoader, ClusterClient clusterClien } catch (Exception e) { String exceptionString = ExceptionUtils.stringifyException(e); if (!(exceptionString.matches("(.*\n)*.*savepoint for the job .* failed(.*\n)*") // legacy + || exceptionString.matches("(.*\n)*.*was not running(.*\n)*") || exceptionString.matches("(.*\n)*.*Not all required tasks are currently running(.*\n)*") // new || exceptionString.matches("(.*\n)*.*Checkpoint was declined \\(tasks not ready\\)(.*\n)*"))) { // new throw e; @@ -151,7 +152,7 @@ private String migrateJob(ClassLoader classLoader, ClusterClient clusterClien assertNotNull("Could not take savepoint.", savepointPath); - CompletableFuture jobCanceledFuture = FutureUtils.retrySuccesfulWithDelay( + CompletableFuture jobCanceledFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobToMigrate.getJobID()), Time.milliseconds(50), deadline, @@ -172,7 +173,7 @@ private void restoreJob(ClassLoader classLoader, ClusterClient clusterClient, clusterClient.submitJob(jobToRestore, classLoader); - CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + CompletableFuture jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobToRestore.getJobID()), Time.milliseconds(50), deadline, diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-rocksdb-savepoint/_metadata index aed61836bd9e2..2e02f1bf87daf 100644 Binary files a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-rocksdb-savepoint/_metadata and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-rocksdb-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-savepoint/_metadata index 13e8d32103623..4ca91d33ccb13 100644 Binary files a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-savepoint/_metadata and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.5-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-rocksdb-savepoint/_metadata index 9e2b08654b28b..8b2a63ab41b24 100644 Binary files a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-rocksdb-savepoint/_metadata and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-rocksdb-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-savepoint/_metadata index cceb5bf7f89b9..cde6d5c44bf6c 100644 Binary files a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-savepoint/_metadata and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.6-savepoint/_metadata differ diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CoGroupGroupSortTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CoGroupGroupSortTranslationTest.scala index a0f93dd5dcb35..1d571d0b3f415 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CoGroupGroupSortTranslationTest.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CoGroupGroupSortTranslationTest.scala @@ -18,22 +18,15 @@ package org.apache.flink.api.scala.operators.translation -import org.apache.flink.api.java.io.DiscardingOutputFormat -import org.apache.flink.optimizer.util.CompilerTestBase -import org.junit.Assert._ -import org.junit.Test -import org.apache.flink.api.common.functions.Partitioner -import org.apache.flink.api.scala._ -import org.apache.flink.runtime.operators.shipping.ShipStrategyType -import org.apache.flink.optimizer.plan.SingleInputPlanNode import org.apache.flink.api.common.operators.Order -import org.apache.flink.api.common.InvalidProgramException -import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint -import org.apache.flink.optimizer.plan.DualInputPlanNode import org.apache.flink.api.common.operators.base.CoGroupOperatorBase -import org.junit.Ignore +import org.apache.flink.api.java.io.DiscardingOutputFormat +import org.apache.flink.api.scala._ +import org.apache.flink.util.{Collector, TestLogger} +import org.junit.Assert._ +import org.junit.{Ignore, Test} -class CoGroupGroupSortTranslationTest { +class CoGroupGroupSortTranslationTest extends TestLogger { @Test def testGroupSortTuples() { @@ -131,11 +124,16 @@ class CoGroupGroupSortTranslationTest { val input2 = env.fromElements( (0L, 0L, 0L) ) input1 - .coGroup(input2) - .where(1).equalTo(2) - .sortFirstGroup(0, Order.DESCENDING) - .sortSecondGroup(1, Order.ASCENDING).sortSecondGroup(0, Order.DESCENDING) - .print() + .coGroup(input2) + .where(1).equalTo(2) + .sortFirstGroup(0, Order.DESCENDING) + .sortSecondGroup(1, Order.ASCENDING).sortSecondGroup(0, Order.DESCENDING) + .apply( + (a: Iterator[(Long, Long)], + b: Iterator[(Long, Long, Long)], + c: Collector[(Long, Long)]) => + a.foreach(e => c.collect(e))) + .output(new DiscardingOutputFormat[(Long, Long)]) val p = env.createProgramPlan() diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index c09b0dd64ca53..7f02ef6f5b9fa 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -185,6 +185,15 @@ under the License. + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + org.apache.maven.plugins maven-shade-plugin diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java index 8e686bbbe34c1..e8e55c3330361 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java @@ -41,15 +41,20 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; +import javax.annotation.Nonnull; + import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -72,6 +77,8 @@ public class YarnFlinkResourceManager extends FlinkResourceManager containersInLaunch; @@ -314,6 +321,21 @@ protected void fatalError(String message, Throwable error) { @Override protected void requestNewWorkers(int numWorkers) { + final Resource capability = getContainerResource(); + + for (int i = 0; i < numWorkers; i++) { + numPendingContainerRequests++; + LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", + capability.getMemory(), numPendingContainerRequests); + + resourceManagerClient.addContainerRequest(createContainerRequest(capability)); + } + + // make sure we transmit the request fast and receive fast news of granted allocations + resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); + } + + private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; @@ -325,25 +347,15 @@ protected void requestNewWorkers(int numWorkers) { mem, containerMemorySizeMB); } - for (int i = 0; i < numWorkers; i++) { - numPendingContainerRequests++; - LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", - containerMemorySizeMB, numPendingContainerRequests); - - // Priority for worker containers - priorities are intra-application - Priority priority = Priority.newInstance(0); - - // Resource requirements for worker containers - int taskManagerSlots = taskManagerParameters.numSlots(); - int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); - Resource capability = Resource.newInstance(containerMemorySizeMB, vcores); - - resourceManagerClient.addContainerRequest( - new AMRMClient.ContainerRequest(capability, null, null, priority)); - } + // Resource requirements for worker containers + int taskManagerSlots = taskManagerParameters.numSlots(); + int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); + return Resource.newInstance(containerMemorySizeMB, vcores); + } - // make sure we transmit the request fast and receive fast news of granted allocations - resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); + @Nonnull + private AMRMClient.ContainerRequest createContainerRequest(Resource capability) { + return new AMRMClient.ContainerRequest(capability, null, null, RM_REQUEST_PRIORITY); } @Override @@ -434,7 +446,14 @@ private void containersAllocated(List containers) { final int numRequired = getDesignatedWorkerPoolSize(); final int numRegistered = getNumberOfStartedTaskManagers(); + final Collection pendingRequests = getPendingRequests(); + final Iterator pendingRequestsIterator = pendingRequests.iterator(); + for (Container container : containers) { + if (numPendingContainerRequests > 0) { + numPendingContainerRequests -= 1; + resourceManagerClient.removeContainerRequest(pendingRequestsIterator.next()); + } numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1); LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); @@ -487,6 +506,24 @@ private void containersAllocated(List containers) { triggerCheckWorkers(); } + private Collection getPendingRequests() { + final List> matchingRequests = resourceManagerClient.getMatchingRequests(RM_REQUEST_PRIORITY, ResourceRequest.ANY, getContainerResource()); + + final Collection result; + + if (matchingRequests.isEmpty()) { + result = Collections.emptyList(); + } else { + result = new ArrayList<>(matchingRequests.get(0)); + } + + Preconditions.checkState( + result.size() == numPendingContainerRequests, + "The RMClient's and YarnResourceManagers internal state about the number of pending container requests has diverged. Number client's pending container requests %s != Number RM's pending container requests %s.", result.size(), numPendingContainerRequests); + + return result; + } + /** * Invoked when the ResourceManager informs of completed containers. * Called via an actor message by the callback from the ResourceManager client. diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 6ff5cd6648711..609ab42f590f0 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -18,6 +18,7 @@ package org.apache.flink.yarn; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ConfigurationUtils; @@ -41,6 +42,7 @@ import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.Preconditions; import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -52,15 +54,20 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -73,6 +80,7 @@ */ public class YarnResourceManager extends ResourceManager implements AMRMClientAsync.CallbackHandler { + private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1); /** The process environment variables. */ private final Map env; @@ -119,6 +127,8 @@ public class YarnResourceManager extends ResourceManager impleme private final Collection slotsPerWorker; + private final Resource resource; + public YarnResourceManager( RpcService rpcService, String resourceManagerEndpointId, @@ -169,6 +179,7 @@ public YarnResourceManager( this.numberOfTaskSlots = flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots); + this.resource = Resource.newInstance(defaultTaskManagerMemoryMB, defaultCpus); this.slotsPerWorker = createSlotsPerWorker(numberOfTaskSlots); } @@ -292,17 +303,19 @@ protected void internalDeregisterApplication( @Override public Collection startNewWorker(ResourceProfile resourceProfile) { - // Priority for worker containers - priorities are intra-application - //TODO: set priority according to the resource allocated - Priority priority = Priority.newInstance(generatePriority(resourceProfile)); - int mem = resourceProfile.getMemoryInMB() < 0 ? defaultTaskManagerMemoryMB : resourceProfile.getMemoryInMB(); - int vcore = resourceProfile.getCpuCores() < 1 ? defaultCpus : (int) resourceProfile.getCpuCores(); - Resource capability = Resource.newInstance(mem, vcore); - requestYarnContainer(capability, priority); + Preconditions.checkArgument( + ResourceProfile.UNKNOWN.equals(resourceProfile), + "The YarnResourceManager does not support custom ResourceProfiles yet. It assumes that all containers have the same resources."); + requestYarnContainer(); return slotsPerWorker; } + @VisibleForTesting + Resource getContainerResource() { + return resource; + } + @Override public boolean stopWorker(final YarnWorkerNode workerNode) { final Container container = workerNode.getContainer(); @@ -343,8 +356,7 @@ public void onContainersCompleted(final List statuses) { if (yarnWorkerNode != null) { // Container completed unexpectedly ~> start a new one - final Container container = yarnWorkerNode.getContainer(); - requestYarnContainerIfRequired(container.getResource(), yarnWorkerNode.getContainer().getPriority()); + requestYarnContainerIfRequired(); } // Eagerly close the connection with task manager. closeTaskManagerConnection(resourceId, new Exception(containerStatus.getDiagnostics())); @@ -356,6 +368,9 @@ public void onContainersCompleted(final List statuses) { @Override public void onContainersAllocated(List containers) { runAsync(() -> { + final Collection pendingRequests = getPendingRequests(); + final Iterator pendingRequestsIterator = pendingRequests.iterator(); + for (Container container : containers) { log.info( "Received new container: {} - Remaining pending container requests: {}", @@ -363,7 +378,7 @@ public void onContainersAllocated(List containers) { numPendingContainerRequests); if (numPendingContainerRequests > 0) { - numPendingContainerRequests--; + removeContainerRequest(pendingRequestsIterator.next()); final String containerIdStr = container.getId().toString(); final ResourceID resourceId = new ResourceID(containerIdStr); @@ -385,7 +400,7 @@ public void onContainersAllocated(List containers) { workerNodeMap.remove(resourceId); resourceManagerClient.releaseAssignedContainer(container.getId()); // and ask for a new one - requestYarnContainerIfRequired(container.getResource(), container.getPriority()); + requestYarnContainerIfRequired(); } } else { // return the excessive containers @@ -402,6 +417,36 @@ public void onContainersAllocated(List containers) { }); } + private void removeContainerRequest(AMRMClient.ContainerRequest pendingContainerRequest) { + numPendingContainerRequests--; + + log.info("Removing container request {}. Pending container requests {}.", pendingContainerRequest, numPendingContainerRequests); + + resourceManagerClient.removeContainerRequest(pendingContainerRequest); + } + + private Collection getPendingRequests() { + final List> matchingRequests = resourceManagerClient.getMatchingRequests( + RM_REQUEST_PRIORITY, + ResourceRequest.ANY, + getContainerResource()); + + final Collection matchingContainerRequests; + + if (matchingRequests.isEmpty()) { + matchingContainerRequests = Collections.emptyList(); + } else { + final Collection collection = matchingRequests.get(0); + matchingContainerRequests = new ArrayList<>(collection); + } + + Preconditions.checkState( + matchingContainerRequests.size() == numPendingContainerRequests, + "The RMClient's and YarnResourceManagers internal state about the number of pending container requests has diverged. Number client's pending container requests %s != Number RM's pending container requests %s.", matchingContainerRequests.size(), numPendingContainerRequests); + + return matchingContainerRequests; + } + @Override public void onShutdownRequest() { shutDown(); @@ -456,17 +501,17 @@ private static Tuple2 parseHostPort(String address) { /** * Request new container if pending containers cannot satisfies pending slot requests. */ - private void requestYarnContainerIfRequired(Resource resource, Priority priority) { + private void requestYarnContainerIfRequired() { int requiredTaskManagerSlots = getNumberRequiredTaskManagerSlots(); int pendingTaskManagerSlots = numPendingContainerRequests * numberOfTaskSlots; if (requiredTaskManagerSlots > pendingTaskManagerSlots) { - requestYarnContainer(resource, priority); + requestYarnContainer(); } } - private void requestYarnContainer(Resource resource, Priority priority) { - resourceManagerClient.addContainerRequest(new AMRMClient.ContainerRequest(resource, null, null, priority)); + private void requestYarnContainer() { + resourceManagerClient.addContainerRequest(getContainerRequest()); // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); @@ -478,6 +523,16 @@ private void requestYarnContainer(Resource resource, Priority priority) { numPendingContainerRequests); } + @Nonnull + @VisibleForTesting + AMRMClient.ContainerRequest getContainerRequest() { + return new AMRMClient.ContainerRequest( + getContainerResource(), + null, + null, + RM_REQUEST_PRIORITY); + } + private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception { // init the ContainerLaunchContext @@ -514,22 +569,4 @@ private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource .put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext; } - - - - /** - * Generate priority by given resource profile. - * Priority is only used for distinguishing request of different resource. - * @param resourceProfile The resource profile of a request - * @return The priority of this resource profile. - */ - private int generatePriority(ResourceProfile resourceProfile) { - if (resourcePriorities.containsKey(resourceProfile)) { - return resourcePriorities.get(resourceProfile); - } else { - int priority = resourcePriorities.size(); - resourcePriorities.put(resourceProfile, priority); - return priority; - } - } } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtils.java index 5566963cb3592..af7bffaae2c5b 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtils.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtils.java @@ -98,7 +98,7 @@ public static Configuration loadConfiguration(String workingDirectory, Map= 0) { + if (!configuration.contains(RestOptions.PORT)) { // set the REST port to 0 to select it randomly configuration.setInteger(RestOptions.PORT, 0); } diff --git a/flink-yarn/src/main/resources/log4j.properties b/flink-yarn/src/main/resources/log4j.properties index b2ad0d3c465bc..e84cd49f01aea 100644 --- a/flink-yarn/src/main/resources/log4j.properties +++ b/flink-yarn/src/main/resources/log4j.properties @@ -17,7 +17,7 @@ ################################################################################ # Convenience file for local debugging of the JobManager/TaskManager. -log4j.rootLogger=INFO, console +log4j.rootLogger=OFF, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTestS3ITCase.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTestS3ITCase.java index e1e95b1c379d1..0fa9893c05438 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTestS3ITCase.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTestS3ITCase.java @@ -23,10 +23,10 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; +import org.apache.flink.testutils.s3.S3TestCredentials; import org.apache.flink.util.TestLogger; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -53,13 +53,8 @@ */ public class YarnFileStageTestS3ITCase extends TestLogger { - private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET"); - private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY"); - @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); @@ -79,9 +74,7 @@ public class YarnFileStageTestS3ITCase extends TestLogger { @BeforeClass public static void checkCredentialsAndSetup() throws IOException { // check whether credentials exist - Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null); + S3TestCredentials.assumeCredentialsAvailable(); skipTest = false; @@ -115,14 +108,14 @@ private static void setupCustomHadoopConfig() throws IOException { Map parameters = new HashMap<>(); // set all different S3 fs implementation variants' configuration keys - parameters.put("fs.s3a.access.key", ACCESS_KEY); - parameters.put("fs.s3a.secret.key", SECRET_KEY); + parameters.put("fs.s3a.access.key", S3TestCredentials.getS3AccessKey()); + parameters.put("fs.s3a.secret.key", S3TestCredentials.getS3SecretKey()); - parameters.put("fs.s3.awsAccessKeyId", ACCESS_KEY); - parameters.put("fs.s3.awsSecretAccessKey", SECRET_KEY); + parameters.put("fs.s3.awsAccessKeyId", S3TestCredentials.getS3AccessKey()); + parameters.put("fs.s3.awsSecretAccessKey", S3TestCredentials.getS3SecretKey()); - parameters.put("fs.s3n.awsAccessKeyId", ACCESS_KEY); - parameters.put("fs.s3n.awsSecretAccessKey", SECRET_KEY); + parameters.put("fs.s3n.awsAccessKeyId", S3TestCredentials.getS3AccessKey()); + parameters.put("fs.s3n.awsSecretAccessKey", S3TestCredentials.getS3SecretKey()); try (PrintStream out = new PrintStream(new FileOutputStream(hadoopConfig))) { out.println(""); @@ -155,7 +148,7 @@ private static void setupCustomHadoopConfig() throws IOException { private void testRecursiveUploadForYarn(String scheme, String pathSuffix) throws Exception { ++numRecursiveUploadTests; - final Path basePath = new Path(scheme + "://" + BUCKET + '/' + TEST_DATA_DIR); + final Path basePath = new Path(S3TestCredentials.getTestBucketUriWithScheme(scheme) + TEST_DATA_DIR); final HadoopFileSystem fs = (HadoopFileSystem) basePath.getFileSystem(); assumeFalse(fs.exists(basePath)); @@ -171,23 +164,6 @@ private void testRecursiveUploadForYarn(String scheme, String pathSuffix) throws } } - /** - * Verifies that nested directories are properly copied with a s3a:// file - * systems during resource uploads for YARN. - */ - @Test - public void testRecursiveUploadForYarnS3() throws Exception { - try { - Class.forName("org.apache.hadoop.fs.s3.S3FileSystem"); - } catch (ClassNotFoundException e) { - // not in the classpath, cannot run this test - String msg = "Skipping test because S3FileSystem is not in the class path"; - log.info(msg); - assumeNoException(msg, e); - } - testRecursiveUploadForYarn("s3", "testYarn-s3"); - } - @Test public void testRecursiveUploadForYarnS3n() throws Exception { try { diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java index 10b2ce97d6fe4..4108249548e56 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; @@ -52,7 +54,6 @@ import org.junit.Test; import org.mockito.Matchers; import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.util.ArrayList; import java.util.Collections; @@ -69,7 +70,10 @@ import scala.concurrent.duration.FiniteDuration; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -128,20 +132,6 @@ public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Except containerList.add(mockContainer); } - doAnswer(new Answer() { - int counter = 0; - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - if (counter < containerList.size()) { - callbackHandler.onContainersAllocated( - Collections.singletonList( - containerList.get(counter++) - )); - } - return null; - } - }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); - final CompletableFuture resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture leaderGatewayFuture = new CompletableFuture<>(); @@ -191,6 +181,9 @@ public Object answer(InvocationOnMock invocation) throws Throwable { nodeManagerClient )); + doReturn(Collections.singletonList(Collections.nCopies(numInitialTaskManagers, new AMRMClient.ContainerRequest(Resource.newInstance(1024 * 1024, 1), null, null, Priority.newInstance(0))))) + .when(resourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); + leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); @@ -203,6 +196,8 @@ public Object answer(InvocationOnMock invocation) throws Throwable { resourceManagerGateway.tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); + callbackHandler.onContainersAllocated(containerList); + for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index d41d42d7a05f9..368e95cc76030 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -105,8 +105,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -243,7 +245,7 @@ class Context { final HardwareDescription hardwareDescription = new HardwareDescription(1, 2L, 3L, 4L); // domain objects for test purposes - final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 200); + final ResourceProfile resourceProfile1 = ResourceProfile.UNKNOWN; public ContainerId task = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(1L, 0), 0), 1); @@ -351,7 +353,7 @@ void runTest(RunnableWithException testMethod) throws Exception { } } - private static Container mockContainer(String host, int port, int containerId) { + private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); @@ -365,7 +367,7 @@ private static Container mockContainer(String host, int port, int containerId) { when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); - when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); + when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; @@ -397,7 +399,10 @@ public void testStopWorker() throws Exception { registerSlotRequestFuture.get(); // Callback from YARN when container is allocated. - Container testingContainer = mockContainer("container", 1234, 1); + Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource()); + + doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest()))) + .when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); @@ -492,10 +497,14 @@ public void testOnContainerCompleted() throws Exception { registerSlotRequestFuture.get(); // Callback from YARN when container is allocated. - Container testingContainer = mockContainer("container", 1234, 1); + Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource()); + + doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest()))) + .when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); + verify(mockResourceManagerClient).removeContainerRequest(any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Callback from YARN when container is Completed, pending request can not be fulfilled by pending diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtilsTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtilsTest.java new file mode 100644 index 0000000000000..c06e270f19164 --- /dev/null +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/entrypoint/YarnEntrypointUtilsTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn.entrypoint; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; +import org.apache.flink.runtime.clusterframework.BootstrapTools; +import org.apache.flink.util.TestLogger; + +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Tests for the {@link YarnEntrypointUtils}. + */ +public class YarnEntrypointUtilsTest extends TestLogger { + + private static final Logger LOG = LoggerFactory.getLogger(YarnEntrypointUtilsTest.class); + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + /** + * Tests that the REST ports are correctly set when loading a {@link Configuration} + * with unspecified REST options. + */ + @Test + public void testRestPortOptionsUnspecified() throws IOException { + final Configuration initialConfiguration = new Configuration(); + + final Configuration configuration = loadConfiguration(initialConfiguration); + + // having not specified the ports should set the rest bind port to 0 + assertThat(configuration.getInteger(RestOptions.PORT), is(equalTo(0))); + } + + /** + * Tests that the REST port option is respected if it has been specified. + */ + @Test + public void testRestPortSpecified() throws IOException { + final Configuration initialConfiguration = new Configuration(); + final int port = 1337; + initialConfiguration.setInteger(RestOptions.PORT, port); + + final Configuration configuration = loadConfiguration(initialConfiguration); + + // if the bind port is not specified it should fall back to the rest port + assertThat(configuration.getInteger(RestOptions.PORT), is(equalTo(port))); + } + + @Nonnull + private Configuration loadConfiguration(Configuration initialConfiguration) throws IOException { + final File workingDirectory = TEMPORARY_FOLDER.newFolder(); + final Map env = new HashMap<>(4); + env.put(ApplicationConstants.Environment.NM_HOST.key(), "foobar"); + + BootstrapTools.writeConfiguration(initialConfiguration, new File(workingDirectory, "flink-conf.yaml")); + return YarnEntrypointUtils.loadConfiguration(workingDirectory.getAbsolutePath(), env, LOG); + } + +} diff --git a/flink-yarn/src/test/resources/log4j-test.properties b/flink-yarn/src/test/resources/log4j-test.properties index 2226f68653181..5b1e4ed8e7a74 100644 --- a/flink-yarn/src/test/resources/log4j-test.properties +++ b/flink-yarn/src/test/resources/log4j-test.properties @@ -18,7 +18,7 @@ # Set root logger level to OFF to not flood build logs # set manually to INFO for debugging purposes -log4j.rootLogger=OFF, testlogger +log4j.rootLogger=INFO, testlogger # A1 is set to be a ConsoleAppender. log4j.appender.testlogger=org.apache.log4j.ConsoleAppender diff --git a/licenses-binary/LICENSE-hdrhistogram b/licenses-binary/LICENSE-hdrhistogram new file mode 100644 index 0000000000000..09c38ea51608e --- /dev/null +++ b/licenses-binary/LICENSE-hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/licenses-binary/LICENSE-protobuf b/licenses-binary/LICENSE-protobuf new file mode 100644 index 0000000000000..21645bec9d5d1 --- /dev/null +++ b/licenses-binary/LICENSE-protobuf @@ -0,0 +1,36 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + diff --git a/licenses-binary/LICENSE-re2j b/licenses-binary/LICENSE-re2j new file mode 100644 index 0000000000000..b620ae68fe332 --- /dev/null +++ b/licenses-binary/LICENSE-re2j @@ -0,0 +1,32 @@ +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE-stax2api b/licenses-binary/LICENSE-stax2api new file mode 100644 index 0000000000000..0ed6361699658 --- /dev/null +++ b/licenses-binary/LICENSE-stax2api @@ -0,0 +1,22 @@ +Copyright woodstox stax2api contributors. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE-xmlenc b/licenses-binary/LICENSE-xmlenc new file mode 100644 index 0000000000000..a1256bdf6f8fd --- /dev/null +++ b/licenses-binary/LICENSE-xmlenc @@ -0,0 +1,28 @@ +Copyright 2003-2005, Ernst de Haan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/licenses-binary/LICENSE.angular b/licenses-binary/LICENSE.angular new file mode 100644 index 0000000000000..91f0644936811 --- /dev/null +++ b/licenses-binary/LICENSE.angular @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2010-2018 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/licenses-binary/LICENSE.angular-drag-and-drop-list b/licenses-binary/LICENSE.angular-drag-and-drop-list new file mode 100644 index 0000000000000..1787fc65befe1 --- /dev/null +++ b/licenses-binary/LICENSE.angular-drag-and-drop-list @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Marcel Juenemann +Copyright (c) 2014-2016 Google Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.angular-moment b/licenses-binary/LICENSE.angular-moment new file mode 100644 index 0000000000000..fe9db79b9113a --- /dev/null +++ b/licenses-binary/LICENSE.angular-moment @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Uri Shaked and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses-binary/LICENSE.angular-ui-router b/licenses-binary/LICENSE.angular-ui-router new file mode 100644 index 0000000000000..6413b092d70f7 --- /dev/null +++ b/licenses-binary/LICENSE.angular-ui-router @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses-binary/LICENSE.arpack_combined_all b/licenses-binary/LICENSE.arpack_combined_all new file mode 100644 index 0000000000000..6a6cdbb5100e3 --- /dev/null +++ b/licenses-binary/LICENSE.arpack_combined_all @@ -0,0 +1,8 @@ +Copyright © 2018 The University of Tennessee. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +· Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +· Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer listed in this license in the documentation and/or other materials provided with the distribution. +· Neither the name of the copyright holders nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. in no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. diff --git a/licenses-binary/LICENSE.asm.txt b/licenses-binary/LICENSE.asm.txt new file mode 100644 index 0000000000000..62ffbccb6837a --- /dev/null +++ b/licenses-binary/LICENSE.asm.txt @@ -0,0 +1,31 @@ +ASM: a very small and fast Java bytecode manipulation framework + +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.base64 b/licenses-binary/LICENSE.base64 new file mode 100644 index 0000000000000..31ebc840539c1 --- /dev/null +++ b/licenses-binary/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/licenses-binary/LICENSE.bootstrap b/licenses-binary/LICENSE.bootstrap new file mode 100644 index 0000000000000..6ca0ceecc5d79 --- /dev/null +++ b/licenses-binary/LICENSE.bootstrap @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2011-2018 Twitter, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses-binary/LICENSE.cddlv1.0 b/licenses-binary/LICENSE.cddlv1.0 new file mode 100644 index 0000000000000..468e7516a87ff --- /dev/null +++ b/licenses-binary/LICENSE.cddlv1.0 @@ -0,0 +1,129 @@ +1. Definitions. + +1.1. "Contributor" means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. "Contributor Version" means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. "Covered Software" means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. "Executable" means the Covered Software in any form other than Source Code. + +1.5. "Initial Developer" means the individual or entity that first makes Original Software available under this License. + +1.6. "Larger Work" means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. "License" means this document. + +1.8. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. "Modifications" means the Source Code and Executable form of any of the following: + +A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + +B. Any new file that contains any part of the Original Software or previous Modification; or + +C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. "Original Software" means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. "Patent Claims" means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. "Source Code" means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. "You" (or "Your") means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + +(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + +(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. + +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. + +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. + +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. + +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. + +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. + +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. + +Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. + +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. + +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as "Participant") alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" (as that term is defined at 48 C.F.R. ¤ 252.227-7014(a)(1)) and "commercial computer software documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. diff --git a/licenses-binary/LICENSE.cddlv1.1 b/licenses-binary/LICENSE.cddlv1.1 new file mode 100644 index 0000000000000..75372e3d2c9a4 --- /dev/null +++ b/licenses-binary/LICENSE.cddlv1.1 @@ -0,0 +1,348 @@ +1. Definitions. + + 1.1. "Contributor" means each individual or entity that creates or + contributes to the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Software, prior Modifications used by a Contributor (if any), and + the Modifications made by that particular Contributor. + + 1.3. "Covered Software" means (a) the Original Software, or (b) + Modifications, or (c) the combination of files containing Original + Software with files containing Modifications, in each case including + portions thereof. + + 1.4. "Executable" means the Covered Software in any form other than + Source Code. + + 1.5. "Initial Developer" means the individual or entity that first + makes Original Software available under this License. + + 1.6. "Larger Work" means a work which combines Covered Software or + portions thereof with code not governed by the terms of this License. + + 1.7. "License" means this document. + + 1.8. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means the Source Code and Executable form of + any of the following: + + A. Any file that results from an addition to, deletion from or + modification of the contents of a file containing Original Software + or previous Modifications; + + B. Any new file that contains any part of the Original Software or + previous Modification; or + + C. Any new file that is contributed or otherwise made available + under the terms of this License. + + 1.10. "Original Software" means the Source Code and Executable form + of computer software code that is originally released under this + License. + + 1.11. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.12. "Source Code" means (a) the common form of computer software + code in which modifications are made and (b) associated + documentation included in or with such code. + + 1.13. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, + this License. For legal entities, "You" includes any entity which + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, the Initial Developer + hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer, to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Software (or portions thereof), with or without Modifications, + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of + Original Software, to make, have made, use, practice, sell, and + offer for sale, and/or otherwise dispose of the Original Software + (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on + the date Initial Developer first distributes or otherwise makes the + Original Software available to a third party under the terms of this + License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: (1) for code that You delete from the Original Software, or + (2) for infringements caused by: (i) the modification of the + Original Software, or (ii) the combination of the Original Software + with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, each Contributor hereby + grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof), either on an + unmodified basis, with other Modifications, as Covered Software + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling + of Modifications made by that Contributor either alone and/or in + combination with its Contributor Version (or portions of such + combination), to make, use, sell, offer for sale, have made, and/or + otherwise dispose of: (1) Modifications made by that Contributor (or + portions thereof); and (2) the combination of Modifications made by + that Contributor with its Contributor Version (or portions of such + combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective + on the date Contributor first distributes or otherwise makes the + Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: (1) for any code that Contributor has deleted from the + Contributor Version; (2) for infringements caused by: (i) third + party modifications of Contributor Version, or (ii) the combination + of Modifications made by that Contributor with other software + (except as part of the Contributor Version) or other devices; or (3) + under Patent Claims infringed by Covered Software in the absence of + Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available + in Executable form must also be made available in Source Code form + and that Source Code form must be distributed only under the terms + of this License. You must include a copy of this License with every + copy of the Source Code form of the Covered Software You distribute + or otherwise make available. You must inform recipients of any such + Covered Software in Executable form as to how they can obtain such + Covered Software in Source Code form in a reasonable manner on or + through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are + governed by the terms of this License. You represent that You + believe Your Modifications are Your original creation(s) and/or You + have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that + identifies You as the Contributor of the Modification. You may not + remove or alter any copyright, patent or trademark notices contained + within the Covered Software, or any notices of licensing or any + descriptive text giving attribution to any Contributor or the + Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in + Source Code form that alters or restricts the applicable version of + this License or the recipients' rights hereunder. You may choose to + offer, and to charge a fee for, warranty, support, indemnity or + liability obligations to one or more recipients of Covered Software. + However, you may do so only on Your own behalf, and not on behalf of + the Initial Developer or any Contributor. You must make it + absolutely clear that any such warranty, support, indemnity or + liability obligation is offered by You alone, and You hereby agree + to indemnify the Initial Developer and every Contributor for any + liability incurred by the Initial Developer or such Contributor as a + result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under + the terms of this License or under the terms of a license of Your + choice, which may contain terms different from this License, + provided that You are in compliance with the terms of this License + and that the license for the Executable form does not attempt to + limit or alter the recipient's rights in the Source Code form from + the rights set forth in this License. If You distribute the Covered + Software in Executable form under a different license, You must make + it absolutely clear that any terms which differ from this License + are offered by You alone, not by the Initial Developer or + Contributor. You hereby agree to indemnify the Initial Developer and + every Contributor for any liability incurred by the Initial + Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with + other code not governed by the terms of this License and distribute + the Larger Work as a single product. In such a case, You must make + sure the requirements of this License are fulfilled for the Covered + Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or + new versions of this License from time to time. Each version will be + given a distinguishing version number. Except as provided in Section + 4.3, no one other than the license steward has the right to modify + this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the + Covered Software available under the terms of the version of the + License under which You originally received the Covered Software. If + the Initial Developer includes a notice in the Original Software + prohibiting it from being distributed or otherwise made available + under any subsequent version of the License, You must distribute and + make the Covered Software available under the terms of the version + of the License under which You originally received the Covered + Software. Otherwise, You may also choose to use, distribute or + otherwise make the Covered Software available under the terms of any + subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new + license for Your Original Software, You may create and use a + modified version of this License if You: (a) rename the license and + remove any references to the name of the license steward (except to + note that the license differs from this License); and (b) otherwise + make it clear that the license contains terms which differ from this + License. + +5. DISCLAIMER OF WARRANTY. + + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, + INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE + IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR + NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF + THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE + DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY + OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, + REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN + ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS + AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to + cure such breach within 30 days of becoming aware of the breach. + Provisions which, by their nature, must remain in effect beyond the + termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding + declaratory judgment actions) against Initial Developer or a + Contributor (the Initial Developer or Contributor against whom You + assert such claim is referred to as "Participant") alleging that the + Participant Software (meaning the Contributor Version where the + Participant is a Contributor or the Original Software where the + Participant is the Initial Developer) directly or indirectly + infringes any patent, then any and all rights granted directly or + indirectly to You by such Participant, the Initial Developer (if the + Initial Developer is not the Participant) and all Contributors under + Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice + from Participant terminate prospectively and automatically at the + expiration of such 60 day notice period, unless if within such 60 + day period You withdraw Your claim with respect to the Participant + Software against such Participant either unilaterally or pursuant to + a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant + alleging that the Participant Software directly or indirectly + infringes any patent where such claim is resolved (such as by + license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, + all end user licenses that have been validly granted by You or any + distributor hereunder prior to termination (excluding licenses + granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE + INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF + COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE + TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR + CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT + LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER + FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR + LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE + POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT + APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH + PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH + LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR + LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION + AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + + The Covered Software is a "commercial item," as that term is defined + in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" (as that term is defined at 48 C.F.R. § + 252.227-7014(a)(1)) and "commercial computer software documentation" + as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent + with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 + (June 1995), all U.S. Government End Users acquire Covered Software + with only those rights set forth herein. This U.S. Government Rights + clause is in lieu of, and supersedes, any other FAR, DFAR, or other + clause or provision that addresses Government rights in computer + software under this License. + +9. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + the law of the jurisdiction specified in a notice contained within + the Original Software (except to the extent applicable law, if any, + provides otherwise), excluding such jurisdiction's conflict-of-law + provisions. Any litigation relating to this License shall be subject + to the jurisdiction of the courts located in the jurisdiction and + venue specified in a notice contained within the Original Software, + with the losing party responsible for costs, including, without + limitation, court costs and reasonable attorneys' fees and expenses. + The application of the United Nations Convention on Contracts for + the International Sale of Goods is expressly excluded. Any law or + regulation which provides that the language of a contract shall be + construed against the drafter shall not apply to this License. You + agree that You alone are responsible for compliance with the United + States export administration regulations (and the export control + laws and regulation of any other countries) when You use, distribute + or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. diff --git a/licenses-binary/LICENSE.core b/licenses-binary/LICENSE.core new file mode 100644 index 0000000000000..b7d28491e4b6b --- /dev/null +++ b/licenses-binary/LICENSE.core @@ -0,0 +1,49 @@ +Copyright (c) 2013 Samuel Halliday +Copyright (c) 1992-2011 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. +Copyright (c) 2000-2011 The University of California Berkeley. All + rights reserved. +Copyright (c) 2006-2011 The University of Colorado Denver. All rights + reserved. + +$COPYRIGHT$ + +Additional copyrights may follow + +$HEADER$ + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + +- Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +The copyright holders provide no reassurances that the source code +provided does not infringe any patent, copyright, or any other +intellectual property rights of third parties. The copyright holders +disclaim any liability to any recipient for claims brought against +recipient by any third party for infringement of that parties +intellectual property rights. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.d3 b/licenses-binary/LICENSE.d3 new file mode 100644 index 0000000000000..1d9d875edb469 --- /dev/null +++ b/licenses-binary/LICENSE.d3 @@ -0,0 +1,27 @@ +Copyright 2010-2017 Mike Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.dagre b/licenses-binary/LICENSE.dagre new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/licenses-binary/LICENSE.dagre @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.dagre-d3 b/licenses-binary/LICENSE.dagre-d3 new file mode 100644 index 0000000000000..1d64ed68ce64d --- /dev/null +++ b/licenses-binary/LICENSE.dagre-d3 @@ -0,0 +1,19 @@ +Copyright (c) 2013 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses-binary/LICENSE.ev-emitter b/licenses-binary/LICENSE.ev-emitter new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/licenses-binary/LICENSE.ev-emitter @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.font-awesome b/licenses-binary/LICENSE.font-awesome new file mode 100644 index 0000000000000..b9fb2c6e6ef06 --- /dev/null +++ b/licenses-binary/LICENSE.font-awesome @@ -0,0 +1,97 @@ +The MIT license + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + SIL OPEN FONT LICENSE + +Version 1.1 - 26 February 2007 +PREAMBLE + +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. +DEFINITIONS + +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting — in part or in whole — any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. +PERMISSION & CONDITIONS + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. +TERMINATION + +This license becomes null and void if any of the above conditions are +not met. +DISCLAIMER + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/licenses-binary/LICENSE.graphlib b/licenses-binary/LICENSE.graphlib new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/licenses-binary/LICENSE.graphlib @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.grizzled-slf4j b/licenses-binary/LICENSE.grizzled-slf4j new file mode 100644 index 0000000000000..6601c0131eeac --- /dev/null +++ b/licenses-binary/LICENSE.grizzled-slf4j @@ -0,0 +1,11 @@ +Copyright © 2010-2012 Brian M. Clapper. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +Neither the name “clapper.org” nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.imagesloaded b/licenses-binary/LICENSE.imagesloaded new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/licenses-binary/LICENSE.imagesloaded @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.janino b/licenses-binary/LICENSE.janino new file mode 100644 index 0000000000000..ef871e2426218 --- /dev/null +++ b/licenses-binary/LICENSE.janino @@ -0,0 +1,31 @@ +Janino - An embedded Java[TM] compiler + +Copyright (c) 2001-2016, Arno Unkrig +Copyright (c) 2015-2016 TIBCO Software Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of JANINO nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.jline b/licenses-binary/LICENSE.jline new file mode 100644 index 0000000000000..e34763968c262 --- /dev/null +++ b/licenses-binary/LICENSE.jline @@ -0,0 +1,7 @@ +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.jquery b/licenses-binary/LICENSE.jquery new file mode 100644 index 0000000000000..e3dbacb999cef --- /dev/null +++ b/licenses-binary/LICENSE.jquery @@ -0,0 +1,20 @@ +Copyright JS Foundation and other contributors, https://js.foundation/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.jsch b/licenses-binary/LICENSE.jsch new file mode 100644 index 0000000000000..edd491dfbfbf5 --- /dev/null +++ b/licenses-binary/LICENSE.jsch @@ -0,0 +1,30 @@ +JSch 0.0.* was released under the GNU LGPL license. Later, we have switched +over to a BSD-style license. + +------------------------------------------------------------------------------ +Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.jsr166y b/licenses-binary/LICENSE.jsr166y new file mode 100644 index 0000000000000..b1c292b54cb2b --- /dev/null +++ b/licenses-binary/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/licenses-binary/LICENSE.jtransforms b/licenses-binary/LICENSE.jtransforms new file mode 100644 index 0000000000000..89169ecf6d12a --- /dev/null +++ b/licenses-binary/LICENSE.jtransforms @@ -0,0 +1,480 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the MPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + https://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] + +The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.mozilla.org/MPL/ + +Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. + +The Original Code is JTransforms. + +The Initial Developer of the Original Code is +Piotr Wendykier, Emory University. +Portions created by the Initial Developer are Copyright (C) 2007-2009 +the Initial Developer. All Rights Reserved. diff --git a/licenses-binary/LICENSE.jython b/licenses-binary/LICENSE.jython new file mode 100644 index 0000000000000..1c2f7ffeb5bc4 --- /dev/null +++ b/licenses-binary/LICENSE.jython @@ -0,0 +1,17 @@ +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 + +1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Jython") in source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Jython alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2007 Python Software Foundation; All Rights Reserved" are retained in Jython alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on or incorporates Jython or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Jython. + +4. PSF is making Jython available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF JYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF JYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING JYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Jython, Licensee agrees to be bound by the terms and conditions of this License Agreement. diff --git a/licenses-binary/LICENSE.jzlib b/licenses-binary/LICENSE.jzlib new file mode 100644 index 0000000000000..29ad562af0a5e --- /dev/null +++ b/licenses-binary/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/licenses-binary/LICENSE.kryo b/licenses-binary/LICENSE.kryo new file mode 100644 index 0000000000000..e1cd88478edf7 --- /dev/null +++ b/licenses-binary/LICENSE.kryo @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.lodash b/licenses-binary/LICENSE.lodash new file mode 100644 index 0000000000000..e3a30b4eafbab --- /dev/null +++ b/licenses-binary/LICENSE.lodash @@ -0,0 +1,10 @@ +The MIT License + +Copyright JS Foundation and other contributors + +Based on Underscore.js, copyright Jeremy Ashkenas, +DocumentCloud and Investigative Reporters & Editors + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/lodash/lodash \ No newline at end of file diff --git a/licenses-binary/LICENSE.machinist b/licenses-binary/LICENSE.machinist new file mode 100644 index 0000000000000..c4a7a1dfd55ef --- /dev/null +++ b/licenses-binary/LICENSE.machinist @@ -0,0 +1,19 @@ +Copyright (c) 2011-2014 Erik Osheim, Tom Switzer + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses-binary/LICENSE.minlog b/licenses-binary/LICENSE.minlog new file mode 100644 index 0000000000000..e1cd88478edf7 --- /dev/null +++ b/licenses-binary/LICENSE.minlog @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.moment b/licenses-binary/LICENSE.moment new file mode 100644 index 0000000000000..8618b7333d6f5 --- /dev/null +++ b/licenses-binary/LICENSE.moment @@ -0,0 +1,22 @@ +Copyright (c) JS Foundation and other contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.moment-duration-format b/licenses-binary/LICENSE.moment-duration-format new file mode 100644 index 0000000000000..06ec6fbe07368 --- /dev/null +++ b/licenses-binary/LICENSE.moment-duration-format @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 vin-car + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses-binary/LICENSE.paranamer b/licenses-binary/LICENSE.paranamer new file mode 100644 index 0000000000000..522176fbca03e --- /dev/null +++ b/licenses-binary/LICENSE.paranamer @@ -0,0 +1,28 @@ +[ ParaNamer used to be 'Pubic Domain', but since it includes a small piece of ASM it is now the same license as that: BSD ] + + Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.protobuf b/licenses-binary/LICENSE.protobuf new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/licenses-binary/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/licenses-binary/LICENSE.qtip2 b/licenses-binary/LICENSE.qtip2 new file mode 100644 index 0000000000000..0cfb790dc91d3 --- /dev/null +++ b/licenses-binary/LICENSE.qtip2 @@ -0,0 +1,22 @@ +Copyright (c) 2012 Craig Michael Thompson + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.scala b/licenses-binary/LICENSE.scala new file mode 100644 index 0000000000000..6d8bdabbbe0fb --- /dev/null +++ b/licenses-binary/LICENSE.scala @@ -0,0 +1,11 @@ +Copyright (c) 2002- EPFL +Copyright (c) 2011- Lightbend, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +Neither the name of the EPFL nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.scopt b/licenses-binary/LICENSE.scopt new file mode 100644 index 0000000000000..6d6a875878f4b --- /dev/null +++ b/licenses-binary/LICENSE.scopt @@ -0,0 +1,21 @@ +This project is licensed under the [MIT license](https://en.wikipedia.org/wiki/MIT_License). + +Copyright (c) scopt contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.slf4j b/licenses-binary/LICENSE.slf4j new file mode 100644 index 0000000000000..93119e70ed5e7 --- /dev/null +++ b/licenses-binary/LICENSE.slf4j @@ -0,0 +1,21 @@ +Copyright (c) 2004-2017 QOS.ch + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.slf4j-api b/licenses-binary/LICENSE.slf4j-api new file mode 100644 index 0000000000000..93119e70ed5e7 --- /dev/null +++ b/licenses-binary/LICENSE.slf4j-api @@ -0,0 +1,21 @@ +Copyright (c) 2004-2017 QOS.ch + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.spire b/licenses-binary/LICENSE.spire new file mode 100644 index 0000000000000..645cba601fd67 --- /dev/null +++ b/licenses-binary/LICENSE.spire @@ -0,0 +1,19 @@ +Copyright (c) 2011-2012 Erik Osheim, Tom Switzer + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses-binary/LICENSE.split b/licenses-binary/LICENSE.split new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/licenses-binary/LICENSE.split @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/licenses-binary/LICENSE.webbit b/licenses-binary/LICENSE.webbit new file mode 100644 index 0000000000000..ec5f348998bd4 --- /dev/null +++ b/licenses-binary/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/licenses-binary/LICENSE.xmlenc b/licenses-binary/LICENSE.xmlenc new file mode 100644 index 0000000000000..3a70c9bfcdadd --- /dev/null +++ b/licenses-binary/LICENSE.xmlenc @@ -0,0 +1,27 @@ +Copyright 2003-2005, Ernst de Haan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/LICENSE.angular b/licenses/LICENSE.angular new file mode 100644 index 0000000000000..91f0644936811 --- /dev/null +++ b/licenses/LICENSE.angular @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2010-2018 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/licenses/LICENSE.angular-drag-and-drop-list b/licenses/LICENSE.angular-drag-and-drop-list new file mode 100644 index 0000000000000..1787fc65befe1 --- /dev/null +++ b/licenses/LICENSE.angular-drag-and-drop-list @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Marcel Juenemann +Copyright (c) 2014-2016 Google Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE.angular-moment b/licenses/LICENSE.angular-moment new file mode 100644 index 0000000000000..fe9db79b9113a --- /dev/null +++ b/licenses/LICENSE.angular-moment @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Uri Shaked and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses/LICENSE.angular-ui-router b/licenses/LICENSE.angular-ui-router new file mode 100644 index 0000000000000..6413b092d70f7 --- /dev/null +++ b/licenses/LICENSE.angular-ui-router @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses/LICENSE.bootstrap b/licenses/LICENSE.bootstrap new file mode 100644 index 0000000000000..6ca0ceecc5d79 --- /dev/null +++ b/licenses/LICENSE.bootstrap @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2011-2018 Twitter, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses/LICENSE.d3 b/licenses/LICENSE.d3 new file mode 100644 index 0000000000000..1d9d875edb469 --- /dev/null +++ b/licenses/LICENSE.d3 @@ -0,0 +1,27 @@ +Copyright 2010-2017 Mike Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/LICENSE.dagre b/licenses/LICENSE.dagre new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/licenses/LICENSE.dagre @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE.dagre-d3 b/licenses/LICENSE.dagre-d3 new file mode 100644 index 0000000000000..1d64ed68ce64d --- /dev/null +++ b/licenses/LICENSE.dagre-d3 @@ -0,0 +1,19 @@ +Copyright (c) 2013 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses/LICENSE.ev-emitter b/licenses/LICENSE.ev-emitter new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/licenses/LICENSE.ev-emitter @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE.font-awesome b/licenses/LICENSE.font-awesome new file mode 100644 index 0000000000000..b9fb2c6e6ef06 --- /dev/null +++ b/licenses/LICENSE.font-awesome @@ -0,0 +1,97 @@ +The MIT license + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + SIL OPEN FONT LICENSE + +Version 1.1 - 26 February 2007 +PREAMBLE + +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. +DEFINITIONS + +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting — in part or in whole — any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. +PERMISSION & CONDITIONS + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. +TERMINATION + +This license becomes null and void if any of the above conditions are +not met. +DISCLAIMER + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/licenses/LICENSE.graphlib b/licenses/LICENSE.graphlib new file mode 100644 index 0000000000000..e3c8f95557dbb --- /dev/null +++ b/licenses/LICENSE.graphlib @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE.imagesloaded b/licenses/LICENSE.imagesloaded new file mode 100644 index 0000000000000..3e30b1aeff26a --- /dev/null +++ b/licenses/LICENSE.imagesloaded @@ -0,0 +1,7 @@ +Copyright © 2018 David DeSandro + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE.jquery b/licenses/LICENSE.jquery new file mode 100644 index 0000000000000..e3dbacb999cef --- /dev/null +++ b/licenses/LICENSE.jquery @@ -0,0 +1,20 @@ +Copyright JS Foundation and other contributors, https://js.foundation/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/LICENSE.lodash b/licenses/LICENSE.lodash new file mode 100644 index 0000000000000..e3a30b4eafbab --- /dev/null +++ b/licenses/LICENSE.lodash @@ -0,0 +1,10 @@ +The MIT License + +Copyright JS Foundation and other contributors + +Based on Underscore.js, copyright Jeremy Ashkenas, +DocumentCloud and Investigative Reporters & Editors + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/lodash/lodash \ No newline at end of file diff --git a/licenses/LICENSE.moment b/licenses/LICENSE.moment new file mode 100644 index 0000000000000..8618b7333d6f5 --- /dev/null +++ b/licenses/LICENSE.moment @@ -0,0 +1,22 @@ +Copyright (c) JS Foundation and other contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/LICENSE.moment-duration-format b/licenses/LICENSE.moment-duration-format new file mode 100644 index 0000000000000..06ec6fbe07368 --- /dev/null +++ b/licenses/LICENSE.moment-duration-format @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 vin-car + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/LICENSE.qtip2 b/licenses/LICENSE.qtip2 new file mode 100644 index 0000000000000..0cfb790dc91d3 --- /dev/null +++ b/licenses/LICENSE.qtip2 @@ -0,0 +1,22 @@ +Copyright (c) 2012 Craig Michael Thompson + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/LICENSE.split b/licenses/LICENSE.split new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/licenses/LICENSE.split @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/pom.xml b/pom.xml index 055e64896e55a..5fd305a7514d5 100644 --- a/pom.xml +++ b/pom.xml @@ -429,7 +429,7 @@ under the License. org.apache.commons commons-compress - 1.4.1 + 1.18 @@ -1251,6 +1251,8 @@ under the License. **/packaged_licenses/LICENSE.*.txt + **/licenses/LICENSE* + **/licenses-binary/LICENSE* flink-runtime-web/web-dashboard/package.json @@ -1482,7 +1484,8 @@ under the License. false true ${project.basedir}/target/dependency-reduced-pom.xml - + + * @@ -1498,6 +1501,13 @@ under the License. ** + + + io.netty:netty + + META-INF/LICENSE.txt + + @@ -1510,13 +1520,13 @@ under the License. org.apache.flink:force-shading - + - - - + + Apache Flink + @@ -1556,7 +1566,7 @@ under the License. com.puppycrawl.tools checkstyle - + 8.9 @@ -1728,7 +1738,7 @@ under the License. org.apache.flink ${project.artifactId} - 1.6.1 + 1.6.2 ${project.packaging} diff --git a/tools/releasing/collect_license_files.sh b/tools/releasing/collect_license_files.sh new file mode 100755 index 0000000000000..38b11b5c681e6 --- /dev/null +++ b/tools/releasing/collect_license_files.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This script extracts from all jars in the specified directory the NOTICE files and the +# licenses folders. It then concatenates all NOTICE files and collects the contents of all +# licenses folders in the specified output directory. +# +# This tool can be used to generate a rough skeleton for the binary NOTICE file. Be aware, +# that it does not deduplicate contents. + +set -Eeuo pipefail + +SRC=${1:-.} +DST=${2:-licenses-output} +PWD=$(pwd) +TMP="${DST}/tmp" + +USAGE="collect_license_files " + +if [ "${SRC}" = "-h" ]; then + echo "${USAGE}" + exit 0 +fi + +for i in $(find -L "${SRC}" -name "*.jar") +do + DIR="${TMP}/$(basename -- "$i" .jar)" + mkdir -p "${DIR}" + JAR="${PWD}/${i}" + (cd "${DIR}" && jar xf ${JAR} META-INF/NOTICE META-INF/licenses) +done + +NOTICE="${DST}/NOTICE" +[ -f "${NOTICE}" ] && rm "${NOTICE}" +find "${TMP}" -name "NOTICE" | xargs cat >> "${NOTICE}" + +LICENSES="${DST}/licenses" +[ -f "${LICENSES}" ] && rm -r "" +find "${TMP}" -name "licenses" -type d -exec cp -r -- "{}" "${DST}" \; + +rm -r "${TMP}" diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index 814612791cbb3..444e01b28cee9 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -20,8 +20,8 @@ ## ## Variables with defaults (if not overwritten by environment) ## -SCALA_VERSION=none -HADOOP_VERSION=none +SCALA_VERSION=${SCALA_VERSION:-none} +HADOOP_VERSION=${HADOOP_VERSION:-none} SKIP_GPG=${SKIP_GPG:-false} MVN=${MVN:-mvn} @@ -78,8 +78,7 @@ make_binary_release() { fi # enable release profile here (to check for the maven version) - tools/change-scala-version.sh ${SCALA_VERSION} - $MVN clean package $FLAGS -Prelease -pl flink-shaded-hadoop/flink-shaded-hadoop2-uber,flink-dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests -Dmaven.test.skip=true + $MVN clean package $FLAGS -Prelease -pl flink-dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests cd flink-dist/target/flink-*-bin/ tar czf "${dir_name}.tgz" flink-* @@ -96,27 +95,27 @@ make_binary_release() { cd ${FLINK_DIR} } +HADOOP_CLASSIFIERS=("24" "26" "27" "28") +HADOOP_VERSIONS=("2.4.1" "2.6.5" "2.7.5" "2.8.3") + if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then make_binary_release "" "-DwithoutHadoop" "2.12" - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.12" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.12" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.5" "2.12" - make_binary_release "hadoop28" "-Dhadoop.version=2.8.3" "2.12" + for i in "${!HADOOP_CLASSIFIERS[@]}"; do + make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "2.12" + done make_binary_release "" "-DwithoutHadoop" "2.11" - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.5" "2.11" - make_binary_release "hadoop28" "-Dhadoop.version=2.8.3" "2.11" + for i in "${!HADOOP_CLASSIFIERS[@]}"; do + make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "2.11" + done elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ] then make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11" elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ] then make_binary_release "" "-DwithoutHadoop" "$SCALA_VERSION" - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "$SCALA_VERSION" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "$SCALA_VERSION" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "$SCALA_VERSION" - make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "$SCALA_VERSION" + for i in "${!HADOOP_CLASSIFIERS[@]}"; do + make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "$SCALA_VERSION" + done else make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION" fi diff --git a/tools/travis/nightly.sh b/tools/travis/nightly.sh new file mode 100755 index 0000000000000..c7429316054e3 --- /dev/null +++ b/tools/travis/nightly.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"${HERE}\" && pwd -P)`" # absolutized and normalized +if [ -z "${HERE}" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +SCRIPT=$1 + +ARTIFACTS_DIR="${HERE}/artifacts" + +mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'." ; exit 1; } + +LOG4J_PROPERTIES=tools/log4j-travis.properties + +MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +MVN_COMMON_OPTIONS="-nsu -B -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast" +MVN_COMPILE_OPTIONS="-T1C -DskipTests" + +cp tools/travis/splits/* flink-end-to-end-tests + +COMMIT_HASH=$(git rev-parse HEAD) +echo "Testing branch ${BRANCH} from remote ${REMOTE}. Commit hash: ${COMMIT_HASH}" + +e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',') +MVN_COMPILE="mvn ${MVN_COMMON_OPTIONS} ${MVN_COMPILE_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} clean install -pl ${e2e_modules},flink-dist -am" + +eval "${MVN_COMPILE}" +EXIT_CODE=$? + +if [ $EXIT_CODE == 0 ]; then + printf "\n\n==============================================================================\n" + printf "Running Java end-to-end tests\n" + printf "==============================================================================\n" + + MVN_TEST="mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl ${e2e_modules} -DdistDir=$(readlink -e build-target)" + + eval "${MVN_TEST}" + EXIT_CODE=$? +else + printf "\n\n==============================================================================\n" + printf "Compile failure detected, skipping Java end-to-end tests\n" + printf "==============================================================================\n" +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n\n==============================================================================\n" + printf "Running end-to-end tests\n" + printf "==============================================================================\n" + + FLINK_DIR=build-target flink-end-to-end-tests/${SCRIPT} + + EXIT_CODE=$? +else + printf "\n\n==============================================================================\n" + printf "Compile failure detected, skipping end-to-end tests\n" + printf "==============================================================================\n" +fi + +# Exit code for Travis build success/failure +exit ${EXIT_CODE} diff --git a/tools/travis/setup_maven.sh b/tools/travis/setup_maven.sh new file mode 100755 index 0000000000000..1de4bd90ae970 --- /dev/null +++ b/tools/travis/setup_maven.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +MAVEN_VERSION="3.2.5" +MAVEN_CACHE_DIR=${HOME}/maven_cache +MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION} + +if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then + wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip + unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip" + rm "apache-maven-${MAVEN_VERSION}-bin.zip" +fi + +export M2_HOME="${MAVEN_VERSIONED_DIR}" +export PATH=${M2_HOME}/bin:${PATH} +export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" + +# just in case: clean up the .m2 home and remove invalid jar files +if [ -d "${HOME}/.m2/repository/" ]; then + find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \; +fi + diff --git a/tools/travis/splits/split_checkpoints.sh b/tools/travis/splits/split_checkpoints.sh new file mode 100755 index 0000000000000..5dfe7e344027e --- /dev/null +++ b/tools/travis/splits/split_checkpoints.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Resuming Savepoint (file, async, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file true" +run_test "Resuming Savepoint (file, sync, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file false" +run_test "Resuming Savepoint (file, async, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true" +run_test "Resuming Savepoint (file, sync, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false" +run_test "Resuming Savepoint (file, async, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true" +run_test "Resuming Savepoint (file, sync, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false" +run_test "Resuming Savepoint (rocks, no parallelism change, heap timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks false heap" +run_test "Resuming Savepoint (rocks, scale up, heap timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks false heap" +run_test "Resuming Savepoint (rocks, scale down, heap timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks false heap" +run_test "Resuming Savepoint (rocks, no parallelism change, rocks timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks false rocks" +run_test "Resuming Savepoint (rocks, scale up, rocks timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks false rocks" +run_test "Resuming Savepoint (rocks, scale down, rocks timers) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks false rocks" + +run_test "Resuming Externalized Checkpoint (file, async, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 file true true" +run_test "Resuming Externalized Checkpoint (file, sync, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 file false true" +run_test "Resuming Externalized Checkpoint (file, async, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 4 file true true" +run_test "Resuming Externalized Checkpoint (file, sync, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 4 file false true" +run_test "Resuming Externalized Checkpoint (file, async, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 4 2 file true true" +run_test "Resuming Externalized Checkpoint (file, sync, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 4 2 file false true" +run_test "Resuming Externalized Checkpoint (rocks, non-incremental, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true false" +run_test "Resuming Externalized Checkpoint (rocks, incremental, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true true" +run_test "Resuming Externalized Checkpoint (rocks, non-incremental, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 4 rocks true false" +run_test "Resuming Externalized Checkpoint (rocks, incremental, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 4 rocks true true" +run_test "Resuming Externalized Checkpoint (rocks, non-incremental, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 4 2 rocks true false" +run_test "Resuming Externalized Checkpoint (rocks, incremental, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 4 2 rocks true true" + +run_test "Resuming Externalized Checkpoint after terminal failure (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 file true false true" +run_test "Resuming Externalized Checkpoint after terminal failure (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 file false false true" +run_test "Resuming Externalized Checkpoint after terminal failure (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true false true" +run_test "Resuming Externalized Checkpoint after terminal failure (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true true true" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_container.sh b/tools/travis/splits/split_container.sh new file mode 100755 index 0000000000000..c62905f9a1de9 --- /dev/null +++ b/tools/travis/splits/split_container.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Running Kerberized YARN on Docker test " "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" + +run_test "Run kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_ha.sh b/tools/travis/splits/split_ha.sh new file mode 100755 index 0000000000000..628b716d7cbba --- /dev/null +++ b/tools/travis/splits/split_ha.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh" "skip_check_exceptions" + +run_test "Running HA (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file true false" "skip_check_exceptions" +run_test "Running HA (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file false false" "skip_check_exceptions" +run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false" "skip_check_exceptions" +run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true" "skip_check_exceptions" + +run_test "Running HA per-job cluster (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file true false" "skip_check_exceptions" +run_test "Running HA per-job cluster (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh file false false" "skip_check_exceptions" +run_test "Running HA per-job cluster (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true false" "skip_check_exceptions" +run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_per_job_cluster_datastream.sh rocks true true" "skip_check_exceptions" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_heavy.sh b/tools/travis/splits/split_heavy.sh new file mode 100755 index 0000000000000..e182d733179a4 --- /dev/null +++ b/tools/travis/splits/split_heavy.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_heavy_deployment.sh" "skip_check_exceptions" + +run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_misc.sh b/tools/travis/splits/split_misc.sh new file mode 100755 index 0000000000000..e98c51cd475c9 --- /dev/null +++ b/tools/travis/splits/split_misc.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Flink CLI end-to-end test" "$END_TO_END_DIR/test-scripts/test_cli.sh" + +run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb" +run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" "skip_check_exceptions" + +run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_allround.sh" +run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh" "skip_check_exceptions" +run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh" "skip_check_exceptions" +run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh" "skip_check_exceptions" +run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh s3" "skip_check_exceptions" +run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4" + +run_test "Elasticsearch (v1.7.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1 https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz" +run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz" +run_test "Elasticsearch (v5.1.2) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz" +run_test "Elasticsearch (v6.3.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 6 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.tar.gz" + +run_test "Quickstarts Java nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh java" +run_test "Quickstarts Scala nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh scala" + +run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" + +run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" +run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" + +run_test "SQL Client end-to-end test" "$END_TO_END_DIR/test-scripts/test_sql_client.sh" +run_test "SQL Client end-to-end test for Kafka 0.10" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka010.sh" +run_test "SQL Client end-to-end test for Kafka 0.11" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka011.sh" +run_test "SQL Client end-to-end test for modern Kafka" "$END_TO_END_DIR/test-scripts/test_sql_client_kafka.sh" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_misc_hadoopfree.sh b/tools/travis/splits/split_misc_hadoopfree.sh new file mode 100755 index 0000000000000..6c552bd7940e3 --- /dev/null +++ b/tools/travis/splits/split_misc_hadoopfree.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Flink CLI end-to-end test" "$END_TO_END_DIR/test-scripts/test_cli.sh" + +run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb" +run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" "skip_check_exceptions" + +run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_allround.sh" +run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh" "skip_check_exceptions" +run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh" "skip_check_exceptions" +run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh s3" "skip_check_exceptions" +run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4" + +run_test "Elasticsearch (v1.7.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1 https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz" +run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz" +run_test "Elasticsearch (v5.1.2) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz" +run_test "Elasticsearch (v6.3.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 6 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.tar.gz" + +run_test "Quickstarts Java nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh java" +run_test "Quickstarts Scala nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh scala" + +run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" + +run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" +run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis/splits/split_sticky.sh b/tools/travis/splits/split_sticky.sh new file mode 100755 index 0000000000000..010f887163fd7 --- /dev/null +++ b/tools/travis/splits/split_sticky.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +export END_TO_END_DIR + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh" + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +# Template for adding a test: + +# run_test "" "$END_TO_END_DIR/test-scripts/" + +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false true" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true false" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false true" "skip_check_exceptions" +run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true" "skip_check_exceptions" + +printf "\n[PASS] All tests passed\n" +exit 0 diff --git a/tools/travis_controller.sh b/tools/travis_controller.sh index 7b2c8fd6a011e..39ada67a0b607 100755 --- a/tools/travis_controller.sh +++ b/tools/travis_controller.sh @@ -17,6 +17,12 @@ # limitations under the License. ################################################################################ +echo $M2_HOME +echo $PATH +echo $MAVEN_OPTS + +mvn -version + CACHE_DIR="$HOME/flink_cache" CACHE_BUILD_DIR="$CACHE_DIR/$TRAVIS_BUILD_NUMBER" CACHE_FLINK_DIR="$CACHE_BUILD_DIR/flink" @@ -46,44 +52,7 @@ function deleteOldCaches() { # delete leftover caches from previous builds find "$CACHE_DIR" -mindepth 1 -maxdepth 1 | grep -v "$TRAVIS_BUILD_NUMBER" | deleteOldCaches -function getCurrentStage() { - STAGE_NUMBER=$(echo "$TRAVIS_JOB_NUMBER" | cut -d'.' -f 2) - case $STAGE_NUMBER in - (1) - echo "$STAGE_COMPILE" - ;; - (2) - echo "$STAGE_CORE" - ;; - (3) - echo "$STAGE_LIBRARIES" - ;; - (4) - echo "$STAGE_CONNECTORS" - ;; - (5) - echo "$STAGE_TESTS" - ;; - (6) - echo "$STAGE_MISC" - ;; - (7) - echo "$STAGE_CLEANUP" - ;; - (*) - echo "Invalid stage detected ($STAGE_NUMBER)" - return 1 - ;; - esac - - return 0 -} - -STAGE=$(getCurrentStage) -if [ $? != 0 ]; then - echo "Could not determine current stage." - exit 1 -fi +STAGE=$1 echo "Current stage: \"$STAGE\"" EXIT_CODE=0 @@ -94,6 +63,19 @@ if [ $STAGE == "$STAGE_COMPILE" ]; then $MVN EXIT_CODE=$? + if [ $EXIT_CODE == 0 ]; then + printf "\n\n==============================================================================\n" + printf "Checking scala suffixes\n" + printf "==============================================================================\n" + + ./tools/verify_scala_suffixes.sh + EXIT_CODE=$? + else + printf "\n==============================================================================\n" + printf "Previous build failure detected, skipping scala-suffixes check.\n" + printf "==============================================================================\n" + fi + if [ $EXIT_CODE == 0 ]; then printf "\n\n==============================================================================\n" printf "Checking dependency convergence\n" @@ -181,9 +163,12 @@ elif [ $STAGE != "$STAGE_CLEANUP" ]; then TEST="$STAGE" "./tools/travis_mvn_watchdog.sh" 300 EXIT_CODE=$? -else +elif [ $STAGE == "$STAGE_CLEANUP" ]; then echo "Cleaning up $CACHE_BUILD_DIR" rm -rf "$CACHE_BUILD_DIR" +else + echo "Invalid Stage specified: $STAGE" + exit 1 fi # Exit code for Travis build success/failure diff --git a/tools/verify_scala_suffixes.sh b/tools/verify_scala_suffixes.sh index 4d7b90c44be25..e4099de5fc7b5 100755 --- a/tools/verify_scala_suffixes.sh +++ b/tools/verify_scala_suffixes.sh @@ -79,6 +79,13 @@ in_block=0 block_name="" block_infected=0 +# exclude e2e modules and flink-docs for convenience as they +# a) are not deployed during a release +# b) exist only for dev purposes +# c) no-one should depend on them +e2e_modules=$(find flink-end-to-end-tests -mindeptha 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',') +excluded_modules=\!${e2e_modules//,/,\!},!flink-docs + echo "Analyzing modules for Scala dependencies using 'mvn dependency:tree'." echo "If you haven't built the project, please do so first by running \"mvn clean install -DskipTests\"" @@ -102,14 +109,14 @@ while read line; do block_name="" block_infected=0 elif [[ $in_block -eq 1 ]]; then - echo $line | grep org.scala-lang >/dev/null + echo $line | grep "org.scala-lang" | grep --invert-match "org.scala-lang.*:.*:.*:test" >/dev/null if [ $? -eq 0 ]; then #echo $block_name infected="$block_name $infected" block_infected=1 fi fi -done < <(mvn -o dependency:tree -Dincludes=org.scala-lang | tee /dev/tty) +done < <(mvn -o dependency:tree -Dincludes=org.scala-lang -pl ${excluded_modules} | tee /dev/tty) # deduplicate and sort