diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 62da82a1ee9933..a017732dc525a9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ - + ## What is the purpose of the change diff --git a/.gitignore b/.gitignore index 8fc9fce6fb2d71..20749c2424251a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ tmp *.log .DS_Store build-target +flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/avro/ flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/generated/ flink-runtime-web/web-dashboard/assets/fonts/ flink-runtime-web/web-dashboard/node_modules/ @@ -34,3 +35,4 @@ out/ *.iws tools/flink tools/flink-* +tools/releasing/release diff --git a/.travis.yml b/.travis.yml index f84b8d8b267d73..a7b1eebb4929d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,11 @@ sudo: required dist: trusty cache: + # default timeout is too low + timeout: 600 directories: - $HOME/.m2 + - $HOME/flink_cache # do not cache our own artifacts before_cache: @@ -31,87 +34,20 @@ install: true language: java -# - define unique cache names in case the auto-generated ones are not unique -# (see https://docs.travis-ci.com/user/caching/#Caches-and-build-matrices) -# - See https://issues.apache.org/jira/browse/FLINK-1072 -matrix: - include: - - jdk: "oraclejdk8" - env: - - TEST="core" - - PROFILE="-Dhadoop.version=2.8.3" - - CACHE_NAME=JDK8_H280_CO - - jdk: "oraclejdk8" - env: - - TEST="libraries" - - PROFILE="-Dhadoop.version=2.8.3" - - CACHE_NAME=JDK8_H280_L - - jdk: "oraclejdk8" - env: - - TEST="connectors" - - PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis" - - CACHE_NAME=JDK8_H280_CN - - jdk: "oraclejdk8" - env: - - TEST="tests" - - PROFILE="-Dhadoop.version=2.8.3" - - CACHE_NAME=JDK8_H280_T - - jdk: "oraclejdk8" - env: - - TEST="misc" - - PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws" - - CACHE_NAME=JDK8_H280_M - - jdk: "openjdk8" - env: - - TEST="core" - - PROFILE="-Dhadoop.version=2.4.1" - - CACHE_NAME=JDK8_H241_CO - - jdk: "openjdk8" - env: - - TEST="libraries" - - PROFILE="-Dhadoop.version=2.4.1" - - CACHE_NAME=JDK8_H241_L - - jdk: "openjdk8" - env: - - TEST="connectors" - - PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis" - - CACHE_NAME=JDK8_H241_CN - - jdk: "openjdk8" - env: - - TEST="tests" - - PROFILE="-Dhadoop.version=2.4.1" - - CACHE_NAME=JDK8_H241_T - - jdk: "openjdk8" - env: - - TEST="misc" - - PROFILE="-Dhadoop.version=2.4.1" - - CACHE_NAME=JDK8_H241_M - - jdk: "oraclejdk8" - env: - - TEST="core" - - PROFILE="-Dhadoop.version=2.8.0 -Pflip6" - - CACHE_NAME=JDK8_H280_F6_CO - - jdk: "oraclejdk8" - env: - - TEST="tests" - - PROFILE="-Dhadoop.version=2.8.0 -Pflip6" - - CACHE_NAME=JDK8_H280_F6_T - git: depth: 100 - env: - global: - # Global variable to avoid hanging travis builds when downloading cache archives. - - MALLOC_ARENA_MAX=2 - # Build artifacts like logs (variables for apache/flink repo) - - secure: "gL3QRn6/XyVK+Em9RmVqpM6nbTwlhjK4/JiRYZGGCkBgTq4ZnG+Eq2qKAO22TAsqRSi7g7WAoAhUulPt0SJqH7hjMe0LetbO0izbVXDefwf2PJlsNgBbuFG6604++VUaUEyfPYYw9ADjV59LWG7+B/fjbRsevqRBZ30b1gv/tQQ=" - - secure: "eM9r8IglvnUKctxz/ga6hwGnCpdOvGyYdGj0H/UiNDEx3Lq1A6yp3gChEIXGJqRUXDI5TaIuidunUGY7KHml8urm8eG2Yk2ttxXehZqLpEaOU2jdNJCdLX8tlVfh14T9bxG5AYHQEV3qJUqDFtfXD3whvzuinrm1oEIA3qUxiA8=" - - secure: "EQYDWgJM5ANJ/sAFwmSEwSTOe9CDN/ENyQAr5/ntM67XanhTZj2Amgt9LthCRUU4EEPl/OFUTwNHMpv/+wa3q7dwVFldSIg5wyCndzJSATPyPBVjYgsXIQZVIjsq4TwTyrTteT55V6Oz2+l27Fvung2FPuN83ovswsJePFzMBxI=" + global: + # Global variable to avoid hanging travis builds when downloading cache archives. + - MALLOC_ARENA_MAX=2 + - DOCKER_COMPOSE_VERSION=1.22.0 before_script: - "gem install --no-document --version 0.8.9 faraday " + - "export -f travis_nanoseconds" + - "export -f travis_time_start" + - "export -f travis_time_finish" # Install maven 3.2.5 since trusty uses 3.3.9 for which shading is broken before_install: @@ -122,8 +58,39 @@ before_install: - "export PATH=$M2_HOME/bin:$PATH" - "export MAVEN_OPTS=\"-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS\"" # just in case: clean up the .m2 home and remove invalid jar files - - 'test ! -d $HOME/.m2/repository/ || find $HOME/.m2/repository/ -name "*.jar" -exec sh -c ''if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm {} ; fi'' \;' + - 'test ! -d $HOME/.m2/repository/ || find $HOME/.m2/repository/ -name "*.jar" -exec sh -c ''if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi'' \;' +# Installing the specified docker compose version + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin -# We run mvn and monitor its output. If there is no output for the specified number of seconds, we -# print the stack traces of all running Java processes. -script: "./tools/travis_mvn_watchdog.sh 300" +# When modifying the matrix you also have to modify travis_controller.sh#getCurrentStage +jdk: "oraclejdk8" +jobs: + include: + # main profile + - stage: compile + script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: compile + - stage: test + script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: core + - script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: libraries + - script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: connectors + - script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: tests + - script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: misc + - stage: cleanup + script: ./tools/travis_controller.sh + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: cleanup diff --git a/LICENSE b/LICENSE index f66290ed5eda62..261eeb9e9f8b2b 100644 --- a/LICENSE +++ b/LICENSE @@ -199,152 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -======================================================================= - -Apache Flink subcomponents: - -The Apache Flink project contains subcomponents in the source code -release with separate copyright notices and license terms. Your use of -the source code for the these subcomponents is subject to the terms and -conditions of their respective licenses. - - ------------------------------------------------------------------------ - The MIT License ------------------------------------------------------------------------ - -The Apache Flink project bundles the following files under the MIT License: - - - Angular.JS v1.4.8 (http://angularjs.org) - Copyright (c) 2010-2015 Google, Inc. - - Angular Moment v0.10.3 (http://www.momentjs.com) - Copyright (C) 2013, 2014, 2015, Uri Shaked - - Angular UI Router v0.2.15 (http://angular-ui.github.io) - Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling - - angular-drag-and-drop-lists v1.4.0 (https://github.com/marceljuenemann/angular-drag-and-drop-lists) Copyright (c) 2014 Marcel Juenemann , Copyright (c) 2014-2016 Google Inc. - - Bootstrap v3.3.6 (http://getbootstrap.com) - Copyright (c) 2011-2015 Twitter, Inc - - d3 timeline (https://github.com/jiahuang/d3-timeline) - Copyright (c) 2012-2015 Jia Huang - - dagre v0.7.4 (https://github.com/cpettitt/dagre) - Copyright (c) 2012-2014 Chris Pettitt - - dagre-d3 v0.4.17 (https://github.com/cpettitt/dagre-d3) - Copyright (c) 2013 Chris Pettitt - - EvEmitter v1.0.2 (https://github.com/metafizzy/ev-emitter) - Copyright (C) 2016 David DeSandro - - Font Awesome (code) v4.5.0, v4.6.3 (http://fontawesome.io) - Copyright (c) 2014 Dave Gandy - - graphlib v1.0.7 (https://github.com/cpettitt/graphlib) - Copyright (c) 2012-2014 Chris Pettitt - - imagesloaded v4.1.0 (https://github.com/desandro/imagesloaded) - Copyright (C) 2016 David DeSandro - - JQuery v2.2.0 (http://jquery.com/) - Copyright 2014 jQuery Foundation and other contributors - - lodash v3.10.1 (http://dojofoundation.org) - Copyright 2012-2015 The Dojo Foundation - - moment.js v2.10.6 (http://momentjs.com/docs/) - Copyright (c) 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors - - moment-duration-format v1.3.0 (https://github.com/jsmreese/moment-duration-format) - Copyright (c) 2013 John Madhavan-Reese - - qTip2 v2.2.1 (http://qtip2.com) - Copyright (c) 2012 Craig Michael Thompson - - Split.js v1.0.6 (https://nathancahill.github.io/Split.js/) - Copyright (c) 2015 Nathan Cahill - -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - ------------------------------------------------------------------------ - BSD-style Licenses (BSD-3-clause) ------------------------------------------------------------------------ - -The Apache Flink project bundles the following files under BSD-3-clause licenses: - - D3 v3.5.12 (http://d3js.org/) - Copyright (c) 2010-2015, Michael Bostock - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list - of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - ------------------------------------------------------------------------ - The Open Font License ------------------------------------------------------------------------ - -The Apache Flink project bundles the following fonts under the -Open Font License (OFT) - http://scripts.sil.org/OFL - - - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy - -> fonts in "flink-runtime-web/web-dashboard/web/fonts" - -> fonts in "docs/page/font-awesome/fonts" - ------------------------------------------------------------------------ - The ISC License ------------------------------------------------------------------------ - -The Apache Flink project contains or reuses code that is licensed under the ISC license -from the following projects: - - - simplejmx (http://256stuff.com/sources/simplejmx/) Copyright (c) - Gray Watson - -Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby -granted, provided that this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING -ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE -USE OR PERFORMANCE OF THIS SOFTWARE. - ------------------------------------------------------------------------ - BSD-style Licenses (BSD-2-clause) ------------------------------------------------------------------------ - -The Apache Flink project contains or reuses code that is licensed under the BSD-2-clause license -from the following projects: - - jline-terminal (https://github.com/jline/jline3) Copyright (c) 2002-2016, various authors. - - jline-reader (https://github.com/jline/jline3) Copyright (c) 2002-2016, various authors. - -Redistribution and use in source and binary forms, with or without modification, are permitted -provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions - and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of - conditions and the following disclaimer in the documentation and/or other materials provided - with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/NOTICE b/NOTICE index da67d0e0d9e40a..8fd65076577013 100644 --- a/NOTICE +++ b/NOTICE @@ -4,18 +4,52 @@ Copyright 2014-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -======================================================================= +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) -Apache Flink contains subcomponents with separate copyright notices and -license terms. Your use of the source code for the these subcomponents -is subject to the terms and conditions of their respective licenses. +- nvd3#1.8.4 -See the LICENSE file for a list of subcomponents and dependencies and -their respective licenses. +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) +See bundled license files for details. +- angular:1.4.8 +- angular-drag-and-drop-list:1.4.0 +- angular-moment:0.10.3 +- angular-ui-router:0.2.15 +- bootstrap:3.3.6 +- dagre:0.7.5 +- dagre-d3:0.4.17 +- ev-emitter:1.1.1 +- font-awesome:4.5.0 (CSS) +- graphlib:1.0.7 +- imagesloaded:4.1.4 +- jquery:2.2.0 +- lodash:3.10.1 +- moment:2.10.6 +- moment-duration-format:1.3.0 +- qtip2:2.2.1 +- Split.js:1.0.6 -lodash: ---------- -Copyright 2012-2015 The Dojo Foundation -Based on Underscore.js, copyright 2009-2015 Jeremy Ashkenas, -DocumentCloud and Investigative Reporters & Editors +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- d3:3.5.12 + +This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1). +See bundled license files for details. + +- font-awesome:4.5.0 (Font) (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy + -> fonts in "flink-runtime-web/web-dashboard/web/fonts" + -> fonts in "docs/page/font-awesome/fonts" + +The Apache Flink project contains or reuses code that is licensed under the ISC license from the following projects. + +- simplejmx (http://256stuff.com/sources/simplejmx/) Copyright (c) - Gray Watson + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby +granted, provided that this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING +ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE +USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/NOTICE-binary b/NOTICE-binary new file mode 100644 index 00000000000000..9028f72aaa4f35 --- /dev/null +++ b/NOTICE-binary @@ -0,0 +1,5863 @@ +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.chuusai:shapeless_2.11:2.3.2 +- log4j:log4j:1.2.17 +- net.sf.opencsv:opencsv:2.3 +- org.scalanlp:breeze_2.11:0.13 +- org.scalanlp:breeze-macros_2.11:0.13 +- org.typelevel:macro-compat_2.11:1.1.1 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.github.fommil.netlib:core:1.1.2 +- net.sourceforge.f2j:arpack_combined_all:0.1 + +This project bundles the following dependencies under the MIT/X11 license. +See bundled license files for details. + +- org.slf4j:slf4j-log4j12:1.7.15 +- org.spire-math:spire-macros_2.11:0.13.0 +- org.spire-math:spire_2.11:0.13.0 +- org.typelevel:machinist_2.11:0.6.1 + +This project bundles the following dependencies under the Mozilla Public license. + +- com.github.rwl:jtransforms:2.4.0 (https://sourceforge.net/projects/jtransforms/) + +flink-metrics-datadog +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.squareup.okhttp3:okhttp:3.7.0 +- com.squareup.okio:okio:1.12.0 + +flink-dist +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.code.findbugs:jsr305:1.3.9 +- com.twitter:chill-java:0.7.6 +- com.twitter:chill_2.11:0.7.6 +- com.typesafe:config:1.3.0 +- com.typesafe:ssl-config-core_2.11:0.2.1 +- com.typesafe.akka:akka-actor_2.11:2.4.20 +- com.typesafe.akka:akka-camel_2.11:2.4.20 +- com.typesafe.akka:akka-protobuf_2.11:2.4.20 +- com.typesafe.akka:akka-slf4j_2.11:2.4.20 +- com.typesafe.akka:akka-stream_2.11:2.4.20 +- commons-cli:commons-cli:1.3.1 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- org.apache.camel:camel-core:2.17.7 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-lang3:3.3.2 +- org.apache.commons:commons-math3:3.5 +- org.javassist:javassist:3.19.0-GA +- org.objenesis:objenesis:2.1 +- org.rocksdb:rocksdbjni:5.7.5 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.esotericsoftware.kryo:kryo:2.24.0 +- com.esotericsoftware.minlog:minlog:1.2 +- org.clapper:grizzled-slf4j_2.11:1.3.2 + +The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala. + +- org.scala-lang:scala-compiler:2.11.12 +- org.scala-lang:scala-library:2.11.12 +- org.scala-lang:scala-reflect:2.11.12 +- org.scala-lang.modules:scala-java8-compat_2.11:0.7.0 +- org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4 +- org.scala-lang.modules:scala-xml_2.11:1.0.5 + +This project bundles the following dependencies under the MIT/X11 license. +See bundled license files for details. + +- com.github.scopt:scopt_2.11:3.5.0 +- org.slf4j:slf4j-api:1.7.15 + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles the following dependencies under the Creative Commons CC0 "No Rights Reserved". + +- org.reactivestreams:reactive-streams:1.0.0 + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +flink-runtime +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.typesafe.akka:akka-remote_2.11:2.4.20 +- io.netty:netty:3.10.6.Final +- org.apache.zookeeper:zookeeper:3.4.10 +- org.uncommons.maths:uncommons-maths:1.2.2a + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +flink-shaded-curator +Copyright 2014-2018 The Apache Software Foundation + +- com.google.guava:guava:16.0.1 +- org.apache.curator:curator-client:2.12.0 +- org.apache.curator:curator-framework:2.12.0 +- org.apache.curator:curator-recipes:2.12.0 + +Curator Recipes +Copyright 2011-2017 The Apache Software Foundation + +Curator Framework +Copyright 2011-2017 The Apache Software Foundation + +Curator Client +Copyright 2011-2017 The Apache Software Foundation + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +flink-shaded-jackson-2 +Copyright 2014-2018 The Apache Software Foundation + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +flink-mesos +Copyright 2014-2018 The Apache Software Foundation + +- com.netflix.fenzo:fenzo-core:0.10.1 +- org.apache.mesos:mesos:1.0.1 +- com.fasterxml.jackson.core:jackson-annotations:2.4.0 +- com.fasterxml.jackson.core:jackson-core:2.4.5 +- com.fasterxml.jackson.core:jackson-databind:2.4.5 + +- com.google.protobuf:protobuf-java:2.6.2 + +mesos +Copyright 2016 The Apache Software Foundation + +flink-yarn +Copyright 2014-2018 The Apache Software Foundation + +Camel :: Core +Copyright 2007-2017 The Apache Software Foundation + + ========================================================================= + == NOTICE file corresponding to the section 4 d of == + == the Apache License, Version 2.0, == + == in this case for the Apache Camel distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Please read the different LICENSE files present in the licenses directory of + this distribution. + +Objenesis +Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita + +flink-table +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.esri.geometry:esri-geometry-api:2.0.0 +- com.google.guava:guava:19.0 +- joda-time:joda-time:2.5 +- net.hydromatic:aggdesigner-algorithm:6.0 +- org.apache.calcite:calcite-core:1.17.0 +- org.apache.calcite:calcite-linq4j:1.17.0 +- org.apache.calcite.avatica:avatica-core:1.12.0 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details + +- org.codehaus.janino:janino:3.0.7 +- org.codehaus.janino:commons-compiler:3.0.7 + +Calcite Core +Copyright 2012-2018 The Apache Software Foundation + +Apache Calcite Avatica +Copyright 2012-2018 The Apache Software Foundation + +Calcite Linq4j +Copyright 2012-2018 The Apache Software Foundation + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +flink-runtime-web +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- nvd3#1.8.4 + +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) +See bundled license files for details. + +- angular:1.4.8 +- angular-drag-and-drop-list:1.4.0 +- angular-moment:0.10.3 +- angular-ui-router:0.2.15 +- bootstrap:3.3.6 +- dagre:0.7.5 +- dagre-d3:0.4.17 +- ev-emitter:1.1.1 +- font-awesome:4.5.0 (CSS) +- graphlib:1.0.7 +- imagesloaded:4.1.4 +- jquery:2.2.0 +- lodash:3.10.1 +- moment:2.10.6 +- moment-duration-format:1.3.0 +- qtip2:2.2.1 +- Split.js:1.0.6 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- d3:3.5.12 + +This project bundles the following dependencies under SIL OFL 1.1 license. (https://opensource.org/licenses/OFL-1.1) +See bundled license files for details. + +- font-awesome:4.5.0 (Font) + +flink-swift-fs-hadoop +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-annotations:2.8.1 +- org.apache.hadoop:hadoop-auth:2.8.1 +- org.apache.hadoop:hadoop-client:2.8.1 +- org.apache.hadoop:hadoop-common:2.8.1 +- org.apache.hadoop:hadoop-hdfs:2.8.1 +- org.apache.hadoop:hadoop-hdfs-client:2.8.1 +- org.apache.hadoop:hadoop-openstack:2.8.1 +- org.apache.htrace:htrace-core4:4.0.1-incubating +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- commons-beanutils:commons-beanutils:1.8.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- commons-httpclient:commons-httpclient:3.1 +- com.google.guava:guava:11.0.2 +- com.google.code.gson:gson:2.2.4 +- com.squareup.okhttp:okhttp:2.4.0 +- com.squareup.okio:okio:1.4.0 +- net.minidev:json-smart:1.1.1 +- com.nimbusds:nimbus-jose-jwt:3.9 +- org.mortbay.jetty:jetty-sslengine:6.1.26 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +This project bundles the following dependencies under the BSD License. +See bundled license files for details. + +- xmlenc:xmlenc:0.52 +- com.google.protobuf:protobuf-java:2.5.0 + +This project bundles "net.jcip:jcip-annotations:1.0". +Written by Brian Goetz and Tim Peierls with assistance from members of JCP JSR-166 Expert Group +and released to the public domain, as explained by the Creative Commons public domain license. +https://creativecommons.org/licenses/publicdomain/ + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + +This project bundles org.apache.hadoop:*:2.8.1 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +/** + * Copyright 2008,2009,2010 Massachusetts Institute of Technology. + * All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +And the binary distribution of this product bundles these dependencies under the +following license: +Mockito 1.8.5 +SLF4J 1.7.10 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +-------------------------------------------------------------------------------- + +Copyright jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js.gz +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.0.0 +-------------------------------------------------------------------------------- +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +servlet-api 2.5 +jsp-api 2.1 +Streaming API for XML 1.0 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +Jersey 1.9 +JAXB API bundle for GlassFish V3 2.2.2 +JAXB RI 2.2.3 +-------------------------------------------------------------------------------- +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +ecj-4.3.1.jar +-------------------------------------------------------------------------------- +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +ASM Core 3.2 +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +xmlenc Library 0.52 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +Java Concurrency in Practice book annotations 1.0 +-------------------------------------------------------------------------------- +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +flink-hadoop-fs +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom +* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni), +which has the following notices: +* This product includes software developed by FuseSource Corp. + http://fusesource.com +* This product includes software developed at + Progress Software Corporation and/or its subsidiaries or affiliates. +* This product includes software developed by IBM Corporation and others. + +The binary distribution of this product bundles binaries of +AWS Java SDK 1.10.6, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: + * license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: + * license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: + * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/libdivsufsort/ + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/snappy/ + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * http://code.google.com/p/caliper/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + +The binary distribution of this product bundles binaries of +Commons Codec 1.4, +which has the following notices: + * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + =============================================================================== + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +The binary distribution of this product bundles binaries of +Commons Lang 2.6, +which has the following notices: + * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +The binary distribution of this product bundles binaries of +Apache Log4j 1.2.17, +which has the following notices: + * ResolverUtil.java + Copyright 2005-2006 Tim Fennell + Dumbster SMTP test server + Copyright 2004 Jason Paul Kitchen + TypeUtil.java + Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +The binary distribution of this product bundles binaries of +Java Concurrency in Practice book annotations 1.0, +which has the following notices: + * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative + Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net Any republication or derived work + distributed in source code form must include this copyright and license + notice. + +The binary distribution of this product bundles binaries of +Jetty 6.1.26, +which has the following notices: + * ============================================================== + Jetty Web Container + Copyright 1995-2016 Mort Bay Consulting Pty Ltd. + ============================================================== + + The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd + unless otherwise noted. + + Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + + Jetty may be distributed under either license. + + ------ + Eclipse + + The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + + The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + ------ + Oracle + + The following artifacts are CDDL + GPLv2 with classpath exception. + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + + ------ + Oracle OpenJDK + + If ALPN is used to negotiate HTTP/2 connections, then the following + artifacts may be included in the distribution or downloaded when ALPN + module is selected. + + * java.sun.security.ssl + + These artifacts replace/modify OpenJDK classes. The modififications + are hosted at github and both modified and original are under GPL v2 with + classpath exceptions. + http://openjdk.java.net/legal/gplv2+ce.html + + ------ + OW2 + + The following artifacts are licensed by the OW2 Foundation according to the + terms of http://asm.ow2.org/license.html + + org.ow2.asm:asm-commons + org.ow2.asm:asm + + ------ + Apache + + The following artifacts are ASL2 licensed. + + org.apache.taglibs:taglibs-standard-spec + org.apache.taglibs:taglibs-standard-impl + + ------ + MortBay + + The following artifacts are ASL2 licensed. Based on selected classes from + following Apache Tomcat jars, all ASL2 licensed. + + org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + + org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + ------ + Mortbay + + The following artifacts are CDDL + GPLv2 with classpath exception. + + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + org.eclipse.jetty.toolchain:jetty-schemas + + ------ + Assorted + + The UnixCrypt.java code implements the one way cryptography used by + Unix systems for simple password protection. Copyright 1996 Aki Yoshida, + modified April 2001 by Iris Van den Broeke, Daniel Deville. + Permission to use, copy, modify and distribute UnixCrypt + for non-commercial or commercial purposes and without fee is + granted provided that the copyright notice appears in all copies./ + +The binary distribution of this product bundles binaries of +Snappy for Java 1.0.4.1, +which has the following notices: + * This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + == Contributors == + * Tatu Saloranta + * Providing benchmark suite + * Alec Wysoker + * Performance and memory usage improvement + +The binary distribution of this product bundles binaries of +Xerces2 Java Parser 2.9.1, +which has the following notices: + * ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache Xerces Java distribution. == + ========================================================================= + + Apache Xerces Java + Copyright 1999-2007 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. + - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. + - voluntary contributions made by Paul Eng on behalf of the + Apache Software Foundation that were originally developed at iClick, Inc., + software copyright (c) 1999. + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Net +Copyright 2001-2012 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Digester +Copyright 2001-2008 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2010 The Apache Software Foundation + +htrace-core4 +Copyright 2015 The Apache Software Foundation + +This product currently only contains code developed by authors +of specific components, as identified by the source code files; +if such notes are missing files have been created by +Tatu Saloranta. + +For additional credits (generally to people who reported problems) +see CREDITS file. + +Apache Jakarta HttpClient +Copyright 1999-2007 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +flink-shaded-hadoop2-uber +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.jamesmurty.utils:java-xmlbuilder:0.4 +- commons-beanutils:commons-beanutils:1.9.3 +- commons-cli:commons-cli:1.3.1 +- commons-codec:commons-codec:1.10 +- commons-collections:commons-collections:3.2.2 +- commons-configuration:commons-configuration:1.7 +- commons-daemon:commons-daemon:1.0.13 +- commons-digester:commons-digester:1.8.1 +- commons-io:commons-io:2.4 +- commons-lang:commons-lang:2.6 +- commons-logging:commons-logging:1.1.3 +- commons-net:commons-net:3.1 +- org.apache.avro:avro:1.8.2 +- org.apache.commons:commons-compress:1.18 +- org.apache.commons:commons-math3:3.5 +- org.apache.zookeeper:zookeeper:3.4.10 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 +- org.xerial.snappy:snappy-java:1.1.4 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.jcraft:jsch:0.1.42 +- com.thoughtworks.paranamer:paranamer:2.7 +- xmlenc:xmlenc:0.52 + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.0. +See bundled license files for details. + +- javax.activation:activation:1.1 (http://java.sun.com/products/javabeans/jaf/index.jsp) +- javax.servlet:servlet-api:2.5 (https://repo1.maven.org/maven2/javax/servlet/servlet-api/2.5/) +- javax.xml.stream:stax-api:1.0-2 (http://central.maven.org/maven2/javax/xml/stream/stax-api/1.0-2/) + +This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.1. +See bundled license files for details. + +- com.sun.jersey:jersey-client:1.9 (http://central.maven.org/maven2/com/sun/jersey/jersey-client/1.9/) +- javax.xml.bind:jaxb-api:2.2.2 (https://jaxb.dev.java.net/) + +This project bundles "org.tukaani:xz:1.5". +This Java implementation of XZ has been put into the public domain, thus you can do +whatever you want with it. All the files in the package have been written by Lasse Collin, +but some files are heavily based on public domain code written by Igor Pavlov. + + +flink-shaded-hadoop2 +Copyright 2014-2018 The Apache Software Foundation + +- com.google.guava:guava:11.0.2 +- net.java.dev.jets3t:jets3t:0.9.0 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- commons-httpclient:commons-httpclient:3.1 +- org.codehaus.jackson:jackson-core-asl:1.9.13 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 + +The version of the actual Hadoop dependencies depend on the built. + +- org.apache.hadoop:hadoop-common +- org.apache.hadoop:hadoop-hdfs +- org.apache.hadoop:hadoop-mapreduce-client-core +- org.apache.hadoop:hadoop-yarn-client +- org.apache.hadoop:hadoop-yarn-common +- org.apache.hadoop:hadoop-annotations +- org.apache.hadoop:hadoop-auth +- org.apache.hadoop:hadoop-yarn-api + +- com.google.protobuf.protobuf-java:2.5.0 + +This product currently only contains code developed by authors +of specific components, as identified by the source code files; +if such notes are missing files have been created by +Tatu Saloranta. + +For additional credits (generally to people who reported problems) +see CREDITS file. + +Apache Jakarta HttpClient +Copyright 1999-2007 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Compress +Copyright 2002-2012 The Apache Software Foundation + +Apache Avro +Copyright 2009-2017 The Apache Software Foundation + +Apache Commons CLI +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +Apache Commons Net +Copyright 2001-2012 The Apache Software Foundation + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons Digester +Copyright 2001-2008 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2016 The Apache Software Foundation + +Apache Commons Daemon +Copyright 1999-2013 The Apache Software Foundation + +flink-s3-fs-presto +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.facebook.presto:presto-hive:0.185 +- com.facebook.presto.hadoop:hadoop-apache2:2.7.3-1 +- com.google.guava:guava:21.0 +- io.airlift:configuration:0.148 +- io.airlift:log:0.148 +- io.airlift:stats:0.148 +- io.airlift:units:1.0 +- io.airlift:slice:0.31 +- com.fasterxml.jackson.core:jackson-annotations:2.8.1 +- com.fasterxml.jackson.core:jackson-core:2.8.1 +- com.fasterxml.jackson.core:jackson-databind:2.8.1 +- joda-time:joda-time:2.5 +- org.weakref:jmxutils:1.19 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.9 + + +flink-s3-fs-base +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-aws:3.1.0 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpclient:4.5.3 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- com.amazonaws:aws-java-sdk-core:1.11.271 +- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 +- com.amazonaws:aws-java-sdk-kms:1.11.271 +- com.amazonaws:aws-java-sdk-s3:1.11.271 +- com.amazonaws:jmespath-java:1.11.271 +- software.amazon.ion:ion-java:1.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.6.0 +- com.fasterxml.jackson.core:jackson-core:2.6.7 +- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 +- joda-time:joda-time:2.5 + +flink-fs-hadoop-shaded +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-annotations:3.1.0 +- org.apache.hadoop:hadoop-auth:3.1.0 +- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.3.2 +- commons-lang:commons-lang:2.6 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.3 +- com.google.guava:guava:11.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.7.0 +- com.fasterxml.jackson.core:jackson-core:2.7.8 +- com.fasterxml.jackson.core:jackson-databind:2.7.8 +- com.fasterxml.woodstox:woodstox-core:5.0.3 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +See bundled license files for details. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +See bundled license files for details. + +- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) + +This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: + +The Apache Hadoop project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +For the org.apache.hadoop.util.bloom.* classes: + +/** + * + * Copyright (c) 2005, European Commission project OneLab under contract + * 034819 (http://www.one-lab.org) + * All rights reserved. + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * - Neither the name of the University Catholique de Louvain - UCL + * nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of the Massachusetts Institute of Technology nor + the names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Other portions are under the same license from Intel: +http://sourceforge.net/projects/slicing-by-8/ +/*++ + * + * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + * + * This software program is licensed subject to the BSD License, + * available at http://www.opensource.org/licenses/bsd-license.html + * + * Abstract: The main routine + * + --*/ + +For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, + +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +For hadoop-common-project/hadoop-common/src/main/native/gtest +--------------------------------------------------------------------- +Copyright 2008, Google Inc. +All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles these dependencies under the +following license: +re2j 1.1 +--------------------------------------------------------------------- +(GO license) +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h +--------------------------------------------------------------------- +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +The binary distribution of this product bundles these dependencies under the +following license: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +bootstrap v3.3.6 +broccoli-asset-rev v2.4.2 +broccoli-funnel v1.0.1 +datatables v1.10.8 +em-helpers v0.5.13 +em-table v0.1.6 +ember v2.2.0 +ember-array-contains-helper v1.0.2 +ember-bootstrap v0.5.1 +ember-cli v1.13.13 +ember-cli-app-version v1.0.0 +ember-cli-babel v5.1.6 +ember-cli-content-security-policy v0.4.0 +ember-cli-dependency-checker v1.2.0 +ember-cli-htmlbars v1.0.2 +ember-cli-htmlbars-inline-precompile v0.3.1 +ember-cli-ic-ajax v0.2.1 +ember-cli-inject-live-reload v1.4.0 +ember-cli-jquery-ui v0.0.20 +ember-cli-qunit v1.2.1 +ember-cli-release v0.2.8 +ember-cli-shims v0.0.6 +ember-cli-sri v1.2.1 +ember-cli-test-loader v0.2.1 +ember-cli-uglify v1.2.0 +ember-d3 v0.1.0 +ember-data v2.1.0 +ember-disable-proxy-controllers v1.0.1 +ember-export-application-global v1.0.5 +ember-load-initializers v0.1.7 +ember-qunit v0.4.16 +ember-qunit-notifications v0.1.0 +ember-resolver v2.0.3 +ember-spin-spinner v0.2.3 +ember-truth-helpers v1.2.0 +jquery v2.1.4 +jquery-ui v1.11.4 +loader.js v3.3.0 +momentjs v2.10.6 +qunit v1.19.0 +select2 v4.0.0 +snippet-ss v1.11.0 +spin.js v2.3.2 +Azure Data Lake Store - Java client SDK 2.0.11 +JCodings 1.0.8 +Joni 2.1.2 +Mockito 1.8.5 +JUL to SLF4J bridge 1.7.25 +SLF4J API Module 1.7.25 +SLF4J LOG4J-12 Binding 1.7.25 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 +Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2003-2017 Optimatika + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For: +oj! Algorithms - version 43.0 +-------------------------------------------------------------------------------- + +Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.3.4 +-------------------------------------------------------------------------------- +(HSQL License) +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +Java Servlet API 3.1.0 +servlet-api 2.5 +jsp-api 2.1 +jsr311-api 1.1.1 +Glassfish Jasper 6.1.14 +Servlet Specification 2.5 API 6.1.14 +-------------------------------------------------------------------------------- +(CDDL 1.0) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +jersey-client 1.19 +jersey-core 1.19 +jersey-grizzly2 1.19 +jersey-grizzly2-servlet 1.19 +jersey-json 1.19 +jersey-server 1.19 +jersey-servlet 1.19 +jersey-guice 1.19 +Jersey Test Framework - Grizzly 2 Module 1.19 +JAXB RI 2.2.3 +Java Architecture for XML Binding 2.2.11 +grizzly-framework 2.2.21 +grizzly-http 2.2.21 +grizzly-http-server 2.2.21 +grizzly-http-servlet 2.2.21 +grizzly-rcm 2.2.21 +-------------------------------------------------------------------------------- +(CDDL 1.1) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +Eclipse JDT Core 3.1.1 +-------------------------------------------------------------------------------- +(EPL v1.0) +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +JSch 0.1.51 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +ASM Core 5.0.4 +ASM Commons 5.0.2 +ASM Tree 5.0.2 +-------------------------------------------------------------------------------- +(3-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. +-------------------------------------------------------------------------------- +(2-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +"Java Concurrency in Practice" book annotations 1.0 +-------------------------------------------------------------------------------- +(CCAL v2.5) +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +The binary distribution of this product bundles these dependencies under the +following license: +jamon-runtime 2.4.1 +-------------------------------------------------------------------------------- +(MPL 2.0) + Mozilla Public License + Version 2.0 + +1.1. “Contributor” +means each individual or legal entity that creates, contributes to the creation +of, or owns Covered Software. + +1.2. “Contributor Version” +means the combination of the Contributions of others (if any) used by a +Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” +means Covered Software of a particular Contributor. + +1.4. “Covered Software” +means Source Code Form to which the initial Contributor has attached the notice +in Exhibit A, the Executable Form of such Source Code Form, and Modifications of +such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” +means + +that the initial Contributor has attached the notice described in Exhibit B to +the Covered Software; or + +that the Covered Software was made available under the terms of version 1.1 or +earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” +means any form of the work other than Source Code Form. + +1.7. “Larger Work” +means a work that combines Covered Software with other material, in a separate +file or files, that is not Covered Software. + +1.8. “License” +means this document. + +1.9. “Licensable” +means having the right to grant, to the maximum extent possible, whether at the +time of the initial grant or subsequently, any and all of the rights conveyed by +this License. + +1.10. “Modifications” +means any of the following: + +any file in Source Code Form that results from an addition to, deletion from, or +modification of the contents of Covered Software; or + +any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor +means any patent claim(s), including without limitation, method, process, and +apparatus claims, in any patent Licensable by such Contributor that would be +infringed, but for the grant of the License, by the making, using, selling, +offering for sale, having made, import, or transfer of either its Contributions +or its Contributor Version. + +1.12. “Secondary License” +means either the GNU General Public License, Version 2.0, the GNU Lesser General +Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, +or any later versions of those licenses. + +1.13. “Source Code Form” +means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) +means an individual or a legal entity exercising rights under this License. For +legal entities, “You” includes any entity that controls, is controlled by, +or is under common control with You. For purposes of this definition, +“control” means (a) the power, direct or indirect, to cause the direction or +management of such entity, whether by contract or otherwise, or (b) ownership of +more than fifty percent (50%) of the outstanding shares or beneficial ownership +of such entity. + +2. License Grants and Conditions + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license: + +under intellectual property rights (other than patent or trademark) Licensable +by such Contributor to use, reproduce, make available, modify, display, perform, +distribute, and otherwise exploit its Contributions, either on an unmodified +basis, with Modifications, or as part of a Larger Work; and + +under Patent Claims of such Contributor to make, use, sell, offer for sale, have +made, import, and otherwise transfer either its Contributions or its Contributor +Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution become +effective for each Contribution on the date the Contributor first distributes +such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under this +License. No additional rights or licenses will be implied from the distribution +or licensing of Covered Software under this License. Notwithstanding Section +2.1(b) above, no patent license is granted by a Contributor: + +for any code that a Contributor has removed from Covered Software; or + +for infringements caused by: (i) Your and any other third party’s +modifications of Covered Software, or (ii) the combination of its Contributions +with other software (except as part of its Contributor Version); or + +under Patent Claims infringed by Covered Software in the absence of its +Contributions. + +This License does not grant any rights in the trademarks, service marks, or +logos of any Contributor (except as may be necessary to comply with the notice +requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to distribute +the Covered Software under a subsequent version of this License (see Section +10.2) or under the terms of a Secondary License (if permitted under the terms of +Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its Contributions are +its original creation(s) or it has sufficient rights to grant the rights to its +Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under applicable +copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in +Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under the +terms of this License. You must inform recipients that the Source Code Form of +the Covered Software is governed by the terms of this License, and how they can +obtain a copy of this License. You may not attempt to alter or restrict the +recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +such Covered Software must also be made available in Source Code Form, as +described in Section 3.1, and You must inform recipients of the Executable Form +how they can obtain a copy of such Source Code Form by reasonable means in a +timely manner, at a charge no more than the cost of distribution to the +recipient; and + +You may distribute such Executable Form under the terms of this License, or +sublicense it under different terms, provided that the license for the +Executable Form does not attempt to limit or alter the recipients’ rights in +the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, provided +that You also comply with the requirements of this License for the Covered +Software. If the Larger Work is a combination of Covered Software with a work +governed by one or more Secondary Licenses, and the Covered Software is not +Incompatible With Secondary Licenses, this License permits You to additionally +distribute such Covered Software under the terms of such Secondary License(s), +so that the recipient of the Larger Work may, at their option, further +distribute the Covered Software under the terms of either this License or such +Secondary License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices (including +copyright notices, patent notices, disclaimers of warranty, or limitations of +liability) contained within the Source Code Form of the Covered Software, except +that You may alter any license notices to the extent required to remedy known +factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, indemnity +or liability obligations to one or more recipients of Covered Software. However, +You may do so only on Your own behalf, and not on behalf of any Contributor. You +must make it absolutely clear that any such warranty, support, indemnity, or +liability obligation is offered by You alone, and You hereby agree to indemnify +every Contributor for any liability incurred by such Contributor as a result of +warranty, support, indemnity or liability terms You offer. You may include +additional disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with +respect to some or all of the Covered Software due to statute, judicial order, +or regulation then You must: (a) comply with the terms of this License to the +maximum extent possible; and (b) describe the limitations and the code they +affect. Such description must be placed in a text file included with all +distributions of the Covered Software under this License. Except to the extent +prohibited by statute or regulation, such description must be sufficiently +detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You +fail to comply with any of its terms. However, if You become compliant, then the +rights granted under this License from a particular Contributor are reinstated +(a) provisionally, unless and until such Contributor explicitly and finally +terminates Your grants, and (b) on an ongoing basis, if such Contributor fails +to notify You of the non-compliance by some reasonable means prior to 60 days +after You have come back into compliance. Moreover, Your grants from a +particular Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the first +time You have received notice of non-compliance with this License from such +Contributor, and You become compliant prior to 30 days after Your receipt of the +notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, counter-claims, and +cross-claims) alleging that a Contributor Version directly or indirectly +infringes any patent, then the rights granted to You by any and all Contributors +for the Covered Software under Section 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user +license agreements (excluding distributors and resellers) which have been +validly granted by You or Your distributors under this License prior to +termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without +warranty of any kind, either expressed, implied, or statutory, including, +without limitation, warranties that the Covered Software is free of defects, +merchantable, fit for a particular purpose or non-infringing. The entire risk as +to the quality and performance of the Covered Software is with You. Should any +Covered Software prove defective in any respect, You (not any Contributor) +assume the cost of any necessary servicing, repair, or correction. This +disclaimer of warranty constitutes an essential part of this License. No use of +any Covered Software is authorized under this License except under this +disclaimer. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort (including +negligence), contract, or otherwise, shall any Contributor, or anyone who +distributes Covered Software as permitted above, be liable to You for any +direct, indirect, special, incidental, or consequential damages of any character +including, without limitation, damages for lost profits, loss of goodwill, work +stoppage, computer failure or malfunction, or any and all other commercial +damages or losses, even if such party shall have been informed of the +possibility of such damages. This limitation of liability shall not apply to +liability for death or personal injury resulting from such party’s negligence +to the extent applicable law prohibits such limitation. Some jurisdictions do +not allow the exclusion or limitation of incidental or consequential damages, so +this exclusion and limitation may not apply to You. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a +jurisdiction where the defendant maintains its principal place of business and +such litigation shall be governed by laws of that jurisdiction, without +reference to its conflict-of-law provisions. Nothing in this Section shall +prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter +hereof. If any provision of this License is held to be unenforceable, such +provision shall be reformed only to the extent necessary to make it enforceable. +Any law or regulation which provides that the language of a contract shall be +construed against the drafter shall not be used to construe this License against +a Contributor. + +10. Versions of the License + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section 10.3, +no one other than the license steward has the right to modify or publish new +versions of this License. Each version will be given a distinguishing version +number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version of the +License under which You originally received the Covered Software, or under the +terms of any subsequent version published by the license steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to create a +new license for such software, you may create and use a modified version of this +License if you rename the license and remove any references to the name of the +license steward (except to note that such modified license differs from this +License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + +If You choose to distribute Source Code Form that is Incompatible With Secondary +Licenses under the terms of this version of the License, the notice described in +Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not distributed with this file, You can obtain one +at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined +by the Mozilla Public License, v. 2.0. + +The binary distribution of this product bundles these dependencies under the +following license: +JDOM 1.1 +-------------------------------------------------------------------------------- +/*-- + + Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the disclaimer that follows + these conditions in the documentation and/or other materials + provided with the distribution. + + 3. The name "JDOM" must not be used to endorse or promote products + derived from this software without prior written permission. For + written permission, please contact . + + 4. Products derived from this software may not be called "JDOM", nor + may "JDOM" appear in their name, without prior written permission + from the JDOM Project Management . + + In addition, we request (but do not require) that you include in the + end-user documentation provided with the redistribution and/or in the + software itself an acknowledgement equivalent to the following: + "This product includes software developed by the + JDOM Project (http://www.jdom.org/)." + Alternatively, the acknowledgment may be graphical using the logos + available at http://www.jdom.org/images/logos. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + + This software consists of voluntary contributions made by many + individuals on behalf of the JDOM Project and was originally + created by Jason Hunter and + Brett McLaughlin . For more information + on the JDOM Project, please see . + + */ + +The binary distribution of this product bundles these dependencies under the +following license: +Hbase Server 1.2.4 +-------------------------------------------------------------------------------- +This project bundles a derivative image for our Orca Logo. This image is +available under the Creative Commons By Attribution 3.0 License. + + Creative Commons Legal Code + + Attribution 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + + License + + THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE + COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY + COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS + AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + + BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE + TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY + BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS + CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND + CONDITIONS. + + 1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined above) for the purposes of this + License. + c. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + d. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + e. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + f. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + g. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + h. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + i. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + + 2. Fair Dealing Rights. Nothing in this License is intended to reduce, + limit, or restrict any uses free from copyright or rights arising from + limitations or exceptions that are provided for in connection with the + copyright protection under copyright law or other applicable laws. + + 3. License Grant. Subject to the terms and conditions of this License, + Licensor hereby grants You a worldwide, royalty-free, non-exclusive, + perpetual (for the duration of the applicable copyright) license to + exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + + The above rights may be exercised in all media and formats whether now + known or hereafter devised. The above rights include the right to make + such modifications as are technically necessary to exercise the rights in + other media and formats. Subject to Section 8(f), all rights not expressly + granted by Licensor are hereby reserved. + + 4. Restrictions. The license granted in Section 3 above is expressly made + subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(b), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(b), as requested. + b. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Section 3(b), in the case of an Adaptation, + a credit identifying the use of the Work in the Adaptation (e.g., + "French translation of the Work by Original Author," or "Screenplay + based on original Work by Original Author"). The credit required by + this Section 4 (b) may be implemented in any reasonable manner; + provided, however, that in the case of a Adaptation or Collection, at + a minimum such credit will appear, if a credit for all contributing + authors of the Adaptation or Collection appears, then as part of these + credits and in a manner at least as prominent as the credits for the + other contributing authors. For the avoidance of doubt, You may only + use the credit required by this Section for the purpose of attribution + in the manner set out above and, by exercising Your rights under this + License, You may not implicitly or explicitly assert or imply any + connection with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of You or Your + use of the Work, without the separate, express prior written + permission of the Original Author, Licensor and/or Attribution + Parties. + c. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + + 5. Representations, Warranties and Disclaimer + + UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR + OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY + KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, + INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, + FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF + LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, + WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION + OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + + 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE + LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR + ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES + ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS + BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + + 8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of this License. + + Creative Commons may be contacted at https://creativecommons.org/. +-------------------------------------------------------------------------------- + +For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/AbstractFuture.java and +hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/TimeoutFuture.java + +Copyright (C) 2007 The Guava Authors + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom +* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +AWS SDK for Java - Bundle 1.11.134, +AWS Java SDK for AWS KMS 1.11.134, +AWS Java SDK for Amazon S3 1.11.134, +AWS Java SDK for AWS STS 1.11.134, +JMES Path Query library 1.0, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: + * license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: + * license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: + * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/libdivsufsort/ + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/snappy/ + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * http://code.google.com/p/caliper/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + +This product contains a modified portion of 'JDOM 1.1', which can be obtained at: + + * LICENSE: + * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt + * HOMEPAGE: + * http://www.jdom.org/ + +The binary distribution of this product bundles binaries of +Commons Codec 1.4, +which has the following notices: + * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + =============================================================================== + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +The binary distribution of this product bundles binaries of +Commons Lang 2.6, +which has the following notices: + * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +The binary distribution of this product bundles binaries of +Apache Log4j 1.2.17, +which has the following notices: + * ResolverUtil.java + Copyright 2005-2006 Tim Fennell + Dumbster SMTP test server + Copyright 2004 Jason Paul Kitchen + TypeUtil.java + Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +The binary distribution of this product bundles binaries of +"Java Concurrency in Practice" book annotations 1.0, +which has the following notices: + * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative + Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net Any republication or derived work + distributed in source code form must include this copyright and license + notice. + +The binary distribution of this product bundles binaries of +Jetty :: Http Utility 9.3.19., +Jetty :: IO Utility 9.3.19., +Jetty :: Security 9.3.19., +Jetty :: Server Core 9.3.19., +Jetty :: Servlet Handling 9.3.19., +Jetty :: Utilities 9.3.19., +Jetty :: Utilities :: Ajax, +Jetty :: Webapp Application Support 9.3.19., +Jetty :: XML utilities 9.3.19., +which has the following notices: + * ============================================================== + Jetty Web Container + Copyright 1995-2016 Mort Bay Consulting Pty Ltd. + ============================================================== + + The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd + unless otherwise noted. + + Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + + Jetty may be distributed under either license. + + ------ + Eclipse + + The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + + The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + ------ + Oracle + + The following artifacts are CDDL + GPLv2 with classpath exception. + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + + ------ + Oracle OpenJDK + + If ALPN is used to negotiate HTTP/2 connections, then the following + artifacts may be included in the distribution or downloaded when ALPN + module is selected. + + * java.sun.security.ssl + + These artifacts replace/modify OpenJDK classes. The modififications + are hosted at github and both modified and original are under GPL v2 with + classpath exceptions. + http://openjdk.java.net/legal/gplv2+ce.html + + ------ + OW2 + + The following artifacts are licensed by the OW2 Foundation according to the + terms of http://asm.ow2.org/license.html + + org.ow2.asm:asm-commons + org.ow2.asm:asm + + ------ + Apache + + The following artifacts are ASL2 licensed. + + org.apache.taglibs:taglibs-standard-spec + org.apache.taglibs:taglibs-standard-impl + + ------ + MortBay + + The following artifacts are ASL2 licensed. Based on selected classes from + following Apache Tomcat jars, all ASL2 licensed. + + org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + + org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + ------ + Mortbay + + The following artifacts are CDDL + GPLv2 with classpath exception. + + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + org.eclipse.jetty.toolchain:jetty-schemas + + ------ + Assorted + + The UnixCrypt.java code implements the one way cryptography used by + Unix systems for simple password protection. Copyright 1996 Aki Yoshida, + modified April 2001 by Iris Van den Broeke, Daniel Deville. + Permission to use, copy, modify and distribute UnixCrypt + for non-commercial or commercial purposes and without fee is + granted provided that the copyright notice appears in all copies./ + +The binary distribution of this product bundles binaries of +Snappy for Java 1.0.4.1, +which has the following notices: + * This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + == Contributors == + * Tatu Saloranta + * Providing benchmark suite + * Alec Wysoker + * Performance and memory usage improvement + +The binary distribution of this product bundles binaries of +Xerces2 Java Parser 2.9.1, +which has the following notices: + * ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache Xerces Java distribution. == + ========================================================================= + + Apache Xerces Java + Copyright 1999-2007 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. + - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. + - voluntary contributions made by Paul Eng on behalf of the + Apache Software Foundation that were originally developed at iClick, Inc., + software copyright (c) 1999. + +The binary distribution of this product bundles binaries of +Logback Classic Module 1.1.2, +Logback Core Module 1.1.2, +which has the following notices: + * Logback: the reliable, generic, fast and flexible logging framework. + Copyright (C) 1999-2012, QOS.ch. All rights reserved. + +The binary distribution of this product bundles binaries of +Apache HBase - Annotations 1.2.6, +Apache HBase - Client 1.2.6, +Apache HBase - Common 1.2.6, +Apache HBase - Hadoop Compatibility 1.2.6, +Apache HBase - Hadoop Two Compatibility 1.2.6, +Apache HBase - Prefix Tree 1.2.6, +Apache HBase - Procedure 1.2.6, +Apache HBase - Protocol 1.2.6, +Apache HBase - Server 1.2.6, +which has the following notices: + * Apache HBase + Copyright 2007-2015 The Apache Software Foundation + + -- + This product incorporates portions of the 'Hadoop' project + + Copyright 2007-2009 The Apache Software Foundation + + Licensed under the Apache License v2.0 + -- + Our Orca logo we got here: http://www.vectorfree.com/jumping-orca + It is licensed Creative Commons Attribution 3.0. + See https://creativecommons.org/licenses/by/3.0/us/ + We changed the logo by stripping the colored background, inverting + it and then rotating it some. + + Later we found that vectorfree.com image is not properly licensed. + The original is owned by vectorportal.com. The original was + relicensed so we could use it as Creative Commons Attribution 3.0. + The license is bundled with the download available here: + http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp + -- + This product includes portions of the Bootstrap project v3.0.0 + + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License v2.0 + + This product uses the Glyphicons Halflings icon set. + + http://glyphicons.com/ + + Copyright Jan Kovařík + + Licensed under the Apache License v2.0 as a part of the Bootstrap project. + + -- + This product includes portions of the Guava project v14, specifically + 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' + + Copyright (C) 2007 The Guava Authors + + Licensed under the Apache License, Version 2.0 + +The binary distribution of this product bundles binaries of +Phoenix Core 4.7.0, +which has the following notices: + Apache Phoenix + Copyright 2013-2016 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This also includes: + + The phoenix-spark module has been adapted from the phoenix-spark library + distributed under the terms of the Apache 2 license. Original source copyright: + Copyright 2014 Simply Measured, Inc. + Copyright 2015 Interset Software Inc. + + The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 + (https://pypi.python.org/pypi/python-daemon/). Original source copyright: + # Copyright © 2008–2015 Ben Finney + # Copyright © 2007–2008 Robert Niederreiter, Jens Klein + # Copyright © 2004–2005 Chad J. Schroeder + # Copyright © 2003 Clark Evans + # Copyright © 2002 Noah Spurrier + # Copyright © 2001 Jürgen Hermann + +The binary distribution of this product bundles binaries of +Plexus Cipher: encryption/decryption Component 1.4, +which has the following notices: + * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java + which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 + + The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java + which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 + +The binary distribution of this product bundles binaries of +software.amazon.ion:ion-java 1.0.1, +which has the following notices: + * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +The binary distribution of this product bundles binaries of +joda-time:joda-time:2.9.9 +which has the following notices: + * ============================================================================= + = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = + ============================================================================= + This product includes software developed by + Joda.org (http://www.joda.org/). + +The binary distribution of this product bundles binaries of +Ehcache 3.3.1, +which has the following notices: + * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. + +The binary distribution of this product bundles binaries of +snakeyaml (https://bitbucket.org/asomov/snakeyaml), +which has the following notices: + * Copyright (c) 2008, http://www.snakeyaml.org + +The binary distribution of this product bundles binaries of +swagger-annotations (https://github.com/swagger-api/swagger-core), +which has the following notices: + * Copyright 2016 SmartBear Software + +The binary distribution of this product bundles binaries of +metrics-core 3.2.4 +which has the following notices: + * Copyright 2010-2013 Coda Hale and Yammer, Inc. + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, + LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + +Apache Commons IO +Copyright 2002-2012 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +Apache Commons Logging +Copyright 2003-2013 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +Apache Commons BeanUtils +Copyright 2000-2016 The Apache Software Foundation + +Apache Commons Configuration +Copyright 2001-2017 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2014 The Apache Software Foundation + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +htrace-core4 +Copyright 2016 The Apache Software Foundation + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +force-shading +Copyright 2018 The Apache Software Foundation + +flink-hadoop-fs +Copyright 2014-2018 The Apache Software Foundation + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2014 The Apache Software Foundation + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +flink-metrics-prometheus +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.prometheus:simpleclient:0.3.0 +- io.prometheus:simpleclient_common:0.3.0 +- io.prometheus:simpleclient_httpserver:0.3.0 +- io.prometheus:simpleclient_pushgateway:0.3.0 + +flink-s3-fs-base +Copyright 2014-2018 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.hadoop:hadoop-aws:3.1.0 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpclient:4.5.3 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- com.amazonaws:aws-java-sdk-core:1.11.271 +- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 +- com.amazonaws:aws-java-sdk-kms:1.11.271 +- com.amazonaws:aws-java-sdk-s3:1.11.271 +- com.amazonaws:jmespath-java:1.11.271 +- software.amazon.ion:ion-java:1.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.6.0 +- com.fasterxml.jackson.core:jackson-core:2.6.7 +- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 +- joda-time:joda-time:2.5 + +flink-fs-hadoop-shaded +Copyright 2014-2018 The Apache Software Foundation + +- org.apache.hadoop:hadoop-annotations:3.1.0 +- org.apache.hadoop:hadoop-auth:3.1.0 +- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.3.2 +- commons-lang:commons-lang:2.6 +- commons-collections:commons-collections:3.2.2 +- commons-io:commons-io:2.4 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.3 +- com.google.guava:guava:11.0.2 +- com.fasterxml.jackson.core:jackson-annotations:2.7.0 +- com.fasterxml.jackson.core:jackson-core:2.7.8 +- com.fasterxml.jackson.core:jackson-databind:2.7.8 +- com.fasterxml.woodstox:woodstox-core:5.0.3 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +See bundled license files for details. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +See bundled license files for details. + +- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) + +flink-sql-client +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- org.jline:jline-terminal:3.9.0 +- org.jline:jline-reader:3.9.0 + + +flink-table +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.esri.geometry:esri-geometry-api:2.0.0 +- com.google.guava:guava:19.0 +- joda-time:joda-time:2.5 +- net.hydromatic:aggdesigner-algorithm:6.0 +- org.apache.calcite:calcite-core:1.17.0 +- org.apache.calcite:calcite-linq4j:1.17.0 +- org.apache.calcite.avatica:avatica-core:1.12.0 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details + +- org.codehaus.janino:janino:3.0.7 +- org.codehaus.janino:commons-compiler:3.0.7 + +flink-table-common +Copyright 2014-2018 The Apache Software Foundation + +Calcite Core +Copyright 2012-2018 The Apache Software Foundation + +Apache Calcite Avatica +Copyright 2012-2018 The Apache Software Foundation + +Calcite Linq4j +Copyright 2012-2018 The Apache Software Foundation + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +force-shading +Copyright 2018 The Apache Software Foundation + +flink-cep +Copyright 2014-2018 The Apache Software Foundation + +flink-streaming-python +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php) +See bundled license files for details. + +- org.python:jython-standalone:2.7.1 + +flink-metrics-graphite +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- io.dropwizard.metrics:metrics-core:3.1.5 +- io.dropwizard.metrics:metrics-graphite:3.1.5 + +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +flink-examples-streaming-state-machine +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 + + +flink-connector-kafka-0.10 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-0.9 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-base +Copyright 2014-2019 The Apache Software Foundation + +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + +flink-examples-streaming-twitter +Copyright 2014-2019 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +flink-connector-twitter +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:14.0.1 +- com.twitter:hbc-core:2.2.0 +- com.twitter:joauth:6.0.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation \ No newline at end of file diff --git a/README.md b/README.md index 1e00e6592b3648..34bf2c4a3eb27f 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Learn more about Flink at [http://flink.apache.org/](http://flink.apache.org/) * Custom memory management for efficient and robust switching between in-memory and out-of-core data processing algorithms -* Compatibility layers for Apache Hadoop MapReduce and Apache Storm +* Compatibility layers for Apache Hadoop MapReduce * Integration with YARN, HDFS, HBase, and other components of the Apache Hadoop ecosystem @@ -67,10 +67,10 @@ counts.writeAsCsv(outputPath) Prerequisites for building Flink: -* Unix-like environment (We use Linux, Mac OS X, Cygwin) +* Unix-like environment (we use Linux, Mac OS X, Cygwin) * git -* Maven (we recommend version 3.0.4) -* Java 8 +* Maven (we recommend version 3.2.5) +* Java 8 (Java 9 and 10 are not yet supported) ``` git clone https://github.com/apache/flink.git @@ -100,7 +100,7 @@ The IntelliJ IDE supports Maven out of the box and offers a plugin for Scala dev * IntelliJ download: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/) * IntelliJ Scala Plugin: [http://plugins.jetbrains.com/plugin/?id=1347](http://plugins.jetbrains.com/plugin/?id=1347) -Check out our [Setting up IntelliJ](https://github.com/apache/flink/blob/master/docs/internals/ide_setup.md#intellij-idea) guide for details. +Check out our [Setting up IntelliJ](https://ci.apache.org/projects/flink/flink-docs-master/flinkDev/ide_setup.html#intellij-idea) guide for details. ### Eclipse Scala IDE diff --git a/deploysettings.xml b/deploysettings.xml deleted file mode 100644 index e36d8480d47e66..00000000000000 --- a/deploysettings.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - apache.snapshots.https - ${sonatype_user} - ${sonatype_pw} - - - apache.releases.https - ${sonatype_user} - ${sonatype_pw} - - - - diff --git a/docs/_config.yml b/docs/_config.yml index 2937499f81334a..5a83c9a95e4e45 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,12 +27,15 @@ # we change the version for the complete docs when forking of a release branch # etc. # The full version string as referenced in Maven (e.g. 1.2.1) -version: "1.5-SNAPSHOT" +version: "1.8-SNAPSHOT" # For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot # release this should be the same as the regular version -version_title: "1.5-SNAPSHOT" -version_javadocs: "1.5" +version_title: "1.8-SNAPSHOT" +# Branch on Github for this version +github_branch: "master" +# Plain Scala version is needed for e.g. the Gradle quickstart. +scala_version: "2.11" # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.11" @@ -45,6 +48,9 @@ download_url: "http://flink.apache.org/downloads.html" # please use a protocol relative URL here baseurl: //ci.apache.org/projects/flink/flink-docs-master +stable_baseurl: //ci.apache.org/projects/flink/flink-docs-stable + +javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master # Flag whether this is a stable version or not. Used for the quickstart page. is_stable: false @@ -53,6 +59,9 @@ is_stable: false show_outdated_warning: false previous_docs: + 1.7: http://ci.apache.org/projects/flink/flink-docs-release-1.7 + 1.6: http://ci.apache.org/projects/flink/flink-docs-release-1.6 + 1.5: http://ci.apache.org/projects/flink/flink-docs-release-1.5 1.4: http://ci.apache.org/projects/flink/flink-docs-release-1.4 1.3: http://ci.apache.org/projects/flink/flink-docs-release-1.3 1.2: http://ci.apache.org/projects/flink/flink-docs-release-1.2 diff --git a/docs/_includes/generated/akka_configuration.html b/docs/_includes/generated/akka_configuration.html index afaba6f0fa5c64..787210119dfb7c 100644 --- a/docs/_includes/generated/akka_configuration.html +++ b/docs/_includes/generated/akka_configuration.html @@ -12,11 +12,41 @@ "10 s" Timeout used for all futures and blocking Akka calls. If Flink fails due to timeouts then you should try to increase this value. Timeouts can be caused by slow machines or a congested network. The timeout value requires a time-unit specifier (ms/s/min/h/d). + +
akka.client-socket-worker-pool.pool-size-factor
+ 1.0 + The pool size factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the pool-size-min and pool-size-max values. + + +
akka.client-socket-worker-pool.pool-size-max
+ 2 + Max number of threads to cap factor-based number to. + + +
akka.client-socket-worker-pool.pool-size-min
+ 1 + Min number of threads to cap factor-based number to. +
akka.client.timeout
"60 s" Timeout for all blocking calls on the client side. + +
akka.fork-join-executor.parallelism-factor
+ 2.0 + The parallelism factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the parallelism-min and parallelism-max values. + + +
akka.fork-join-executor.parallelism-max
+ 64 + Max number of threads to cap factor-based parallelism number to. + + +
akka.fork-join-executor.parallelism-min
+ 8 + Min number of threads to cap factor-based parallelism number to. +
akka.framesize
"10485760b" @@ -30,7 +60,7 @@
akka.log.lifecycle.events
false - Turns on the Akka’s remote logging of events. Set this value to ‘true’ in case of debugging. + Turns on the Akka’s remote logging of events. Set this value to 'true' in case of debugging.
akka.lookup.timeout
@@ -42,6 +72,21 @@ 50 Milliseconds a gate should be closed for after a remote connection was disconnected. + +
akka.server-socket-worker-pool.pool-size-factor
+ 1.0 + The pool size factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the pool-size-min and pool-size-max values. + + +
akka.server-socket-worker-pool.pool-size-max
+ 2 + Max number of threads to cap factor-based number to. + + +
akka.server-socket-worker-pool.pool-size-min
+ 1 + Min number of threads to cap factor-based number to. +
akka.ssl.enabled
true @@ -80,17 +125,17 @@
akka.watch.heartbeat.interval
"10 s" - Heartbeat interval for Akka’s DeathWatch mechanism to detect dead TaskManagers. If TaskManagers are wrongly marked dead because of lost or delayed heartbeat messages, then you should decrease this value or increase akka.watch.heartbeat.pause. A thorough description of Akka’s DeathWatch can be found <a href="http://doc.akka.io/docs/akka/snapshot/scala/remoting.html#failure-detector">here</a>. + Heartbeat interval for Akka’s DeathWatch mechanism to detect dead TaskManagers. If TaskManagers are wrongly marked dead because of lost or delayed heartbeat messages, then you should decrease this value or increase akka.watch.heartbeat.pause. A thorough description of Akka’s DeathWatch can be found here
akka.watch.heartbeat.pause
"60 s" - Acceptable heartbeat pause for Akka’s DeathWatch mechanism. A low value does not allow an irregular heartbeat. If TaskManagers are wrongly marked dead because of lost or delayed heartbeat messages, then you should increase this value or decrease akka.watch.heartbeat.interval. Higher value increases the time to detect a dead TaskManager. A thorough description of Akka’s DeathWatch can be found <a href="http://doc.akka.io/docs/akka/snapshot/scala/remoting.html#failure-detector">here</a>. + Acceptable heartbeat pause for Akka’s DeathWatch mechanism. A low value does not allow an irregular heartbeat. If TaskManagers are wrongly marked dead because of lost or delayed heartbeat messages, then you should increase this value or decrease akka.watch.heartbeat.interval. Higher value increases the time to detect a dead TaskManager. A thorough description of Akka’s DeathWatch can be found here
akka.watch.threshold
12 - Threshold for the DeathWatch failure detector. A low value is prone to false positives whereas a high value increases the time to detect a dead TaskManager. A thorough description of Akka’s DeathWatch can be found <a href="http://doc.akka.io/docs/akka/snapshot/scala/remoting.html#failure-detector">here</a>. + Threshold for the DeathWatch failure detector. A low value is prone to false positives whereas a high value increases the time to detect a dead TaskManager. A thorough description of Akka’s DeathWatch can be found here diff --git a/docs/_includes/generated/checkpointing_configuration.html b/docs/_includes/generated/checkpointing_configuration.html index c64112e1f0cf9c..6ad5eeeb091089 100644 --- a/docs/_includes/generated/checkpointing_configuration.html +++ b/docs/_includes/generated/checkpointing_configuration.html @@ -28,14 +28,14 @@ Option whether the state backend should create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Some state backends may not support incremental checkpoints and ignore this option. -
state.backend.rocksdb.localdir
- (none) - The local directory (on the TaskManager) where RocksDB puts its files. +
state.backend.local-recovery
+ false + This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.
state.checkpoints.dir
(none) - The default directory used for checkpoints. Used by the state backends that write checkpoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend). + The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).
state.checkpoints.num-retained
@@ -47,5 +47,10 @@ (none) The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend). + +
taskmanager.state.local.root-dirs
+ (none) + The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option + diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_section.html new file mode 100644 index 00000000000000..1c6685b2f4c657 --- /dev/null +++ b/docs/_includes/generated/common_section.html @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
jobmanager.heap.size
"1024m"JVM heap size for the JobManager.
taskmanager.heap.size
"1024m"JVM heap size for the TaskManagers, which are the parallel workers of the system. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value.
parallelism.default
1Default parallelism for jobs.
taskmanager.numberOfTaskSlots
1The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).
state.backend
(none)The state backend to be used to store and checkpoint state.
state.checkpoints.dir
(none)The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).
state.savepoints.dir
(none)The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).
high-availability
"NONE"Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.
high-availability.storageDir
(none)File system path (URI) where Flink persists metadata in high-availability setups.
security.ssl.internal.enabled
falseTurns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).
security.ssl.rest.enabled
falseTurns on SSL for external communication via the REST endpoints.
diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/core_configuration.html index 520ce05a54102d..540290045a8309 100644 --- a/docs/_includes/generated/core_configuration.html +++ b/docs/_includes/generated/core_configuration.html @@ -8,9 +8,14 @@ -
classloader.parent-first-patterns
- "java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback" - A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. +
classloader.parent-first-patterns.additional
+ (none) + A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. These patterns are appended to "classloader.parent-first-patterns.default". + + +
classloader.parent-first-patterns.default
+ "java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging;org.apache.commons.logging;ch.qos.logback" + A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. This setting should generally not be modified. To add another pattern we recommend to use "classloader.parent-first-patterns.additional" instead.
classloader.resolve-order
@@ -19,18 +24,13 @@
io.tmp.dirs
- (none) - - - -
mode
- "flip6" - Switch to select the execution mode. Possible values are 'flip6' and 'old'. + 'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone. + Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.
parallelism.default
1 - + Default parallelism for jobs. diff --git a/docs/_includes/generated/environment_configuration.html b/docs/_includes/generated/environment_configuration.html index a56a8c245204d2..a54955c7342716 100644 --- a/docs/_includes/generated/environment_configuration.html +++ b/docs/_includes/generated/environment_configuration.html @@ -7,20 +7,30 @@ + +
env.hadoop.conf.dir
+ (none) + Path to hadoop configuration directory. It is required to read HDFS and/or YARN configuration. You can also set it via environment variable. +
env.java.opts
(none) - + Java options to start the JVM of all Flink processes with. + + +
env.java.opts.historyserver
+ (none) + Java options to start the JVM of the HistoryServer with.
env.java.opts.jobmanager
(none) - + Java options to start the JVM of the JobManager with.
env.java.opts.taskmanager
(none) - + Java options to start the JVM of the TaskManager with.
env.log.dir
@@ -37,5 +47,10 @@ (none) Additional command line options passed to SSH clients when starting or stopping JobManager, TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh, stop-zookeeper-quorum.sh). + +
env.yarn.conf.dir
+ (none) + Path to yarn configuration directory. It is required to run flink on YARN. You can also set it via environment variable. + diff --git a/docs/_includes/generated/high_availability_configuration.html b/docs/_includes/generated/high_availability_configuration.html index b1e2ea9062cb52..398379da6351db 100644 --- a/docs/_includes/generated/high_availability_configuration.html +++ b/docs/_includes/generated/high_availability_configuration.html @@ -10,7 +10,7 @@
high-availability
"NONE" - Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER". + Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.
high-availability.cluster-id
diff --git a/docs/_includes/generated/high_availability_zookeeper_configuration.html b/docs/_includes/generated/high_availability_zookeeper_configuration.html index a49d160618a6ad..6577878674b3de 100644 --- a/docs/_includes/generated/high_availability_zookeeper_configuration.html +++ b/docs/_includes/generated/high_availability_zookeeper_configuration.html @@ -60,7 +60,7 @@
high-availability.zookeeper.path.mesos-workers
"/mesos-workers" - ZooKeeper root path (ZNode) for Mesos workers. + The ZooKeeper root path for persisting the Mesos worker information.
high-availability.zookeeper.path.root
diff --git a/docs/_includes/generated/history_server_configuration.html b/docs/_includes/generated/history_server_configuration.html index 272606e2884b68..7691c0abe62a40 100644 --- a/docs/_includes/generated/history_server_configuration.html +++ b/docs/_includes/generated/history_server_configuration.html @@ -30,7 +30,7 @@
historyserver.web.refresh-interval
10000 - + The refresh interval for the HistoryServer web-frontend in milliseconds.
historyserver.web.ssl.enabled
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html index 251731a8e05d24..177c362fb8bf93 100644 --- a/docs/_includes/generated/job_manager_configuration.html +++ b/docs/_includes/generated/job_manager_configuration.html @@ -10,7 +10,7 @@
jobmanager.archive.fs.dir
(none) - + Dictionary for JobManager to store the archives of completed jobs.
jobmanager.execution.attempts-history-size
@@ -18,14 +18,9 @@ The maximum number of prior execution attempts kept in history. -
jobmanager.execution.failover-strategy
- "full" - The maximum number of prior execution attempts kept in history. - - -
jobmanager.heap.mb
- 1024 - JVM heap size (in megabytes) for the JobManager. +
jobmanager.heap.size
+ "1024m" + JVM heap size for the JobManager.
jobmanager.resourcemanager.reconnect-interval
@@ -54,7 +49,7 @@
slot.idle.timeout
- 10000 + 50000 The timeout in milliseconds for a idle slot in Slot Pool. diff --git a/docs/_includes/generated/mesos_configuration.html b/docs/_includes/generated/mesos_configuration.html index 16a23886cee53b..54e92e5680c51e 100644 --- a/docs/_includes/generated/mesos_configuration.html +++ b/docs/_includes/generated/mesos_configuration.html @@ -9,23 +9,23 @@
mesos.failover-timeout
- 600 + 604800 The failover timeout in seconds for the Mesos scheduler, after which running tasks are automatically shut down.
mesos.initial-tasks
0 - The initial workers to bring up when the master starts + The initial workers to bring up when the master starts. This option is ignored unless Flink is in legacy mode.
mesos.master
(none) - The Mesos master URL. The value should be in one of the following forms: "host:port", "zk://host1:port1,host2:port2,.../path", "zk://username:password@host1:port1,host2:port2,.../path" or "file:///path/to/file" + The Mesos master URL. The value should be in one of the following forms:
  • host:port
  • zk://host1:port1,host2:port2,.../path
  • zk://username:password@host1:port1,host2:port2,.../path
  • file:///path/to/file
mesos.maximum-failed-tasks
-1 - The maximum number of failed workers before the cluster fails. May be set to -1 to disable this feature + The maximum number of failed workers before the cluster fails. May be set to -1 to disable this feature. This option is ignored unless Flink is in legacy mode.
mesos.resourcemanager.artifactserver.port
@@ -62,5 +62,10 @@ (none) Mesos framework user + +
mesos.resourcemanager.tasks.port-assignments
+ (none) + Comma-separated list of configuration keys which represent a configurable port. All port keys will dynamically get a port assigned through Mesos. + diff --git a/docs/_includes/generated/mesos_task_manager_configuration.html b/docs/_includes/generated/mesos_task_manager_configuration.html index 182418dd2e110b..1e67f8429d7466 100644 --- a/docs/_includes/generated/mesos_task_manager_configuration.html +++ b/docs/_includes/generated/mesos_task_manager_configuration.html @@ -10,12 +10,17 @@
mesos.constraints.hard.hostattribute
(none) - Constraints for task placement on mesos. + Constraints for task placement on Mesos based on agent attributes. Takes a comma-separated list of key:value pairs corresponding to the attributes exposed by the target mesos agents. Example: az:eu-west-1a,series:t2
mesos.resourcemanager.tasks.bootstrap-cmd
(none) - + A command which is executed before the TaskManager is started. + + +
mesos.resourcemanager.tasks.container.docker.force-pull-image
+ false + Instruct the docker containerizer to forcefully pull the image rather than reuse a cached version.
mesos.resourcemanager.tasks.container.docker.parameters
@@ -45,12 +50,12 @@
mesos.resourcemanager.tasks.gpus
0 - + GPUs to assign to the Mesos workers.
mesos.resourcemanager.tasks.hostname
(none) - + Optional value to define the TaskManager’s hostname. The pattern _TASK_ is replaced by the actual id of the Mesos task. This can be used to configure the TaskManager to use Mesos DNS (e.g. _TASK_.flink-service.mesos) for name lookups.
mesos.resourcemanager.tasks.mem
@@ -62,10 +67,15 @@ "$FLINK_HOME/bin/mesos-taskmanager.sh" + +
mesos.resourcemanager.tasks.uris
+ (none) + A comma separated list of URIs of custom artifacts to be downloaded into the sandbox of Mesos workers. +
taskmanager.numberOfTaskSlots
1 - + The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores). diff --git a/docs/_includes/generated/metric_configuration.html b/docs/_includes/generated/metric_configuration.html index 868a9bd66e9b61..39a76cef165cae 100644 --- a/docs/_includes/generated/metric_configuration.html +++ b/docs/_includes/generated/metric_configuration.html @@ -7,65 +7,95 @@ + +
metrics.internal.query-service.port
+ "0" + The port range used for Flink's internal metric query service. Accepts a list of ports (“50100,50101”), ranges(“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Flink components are running on the same machine. Per default Flink will pick a random port. + + +
metrics.internal.query-service.thread-priority
+ 1 + The thread priority used for Flink's internal metric query service. The thread is created by Akka's thread pool executor. The range of the priority is from 1 (MIN_PRIORITY) to 10 (MAX_PRIORITY). Warning, increasing this value may bring the main Flink components down. + + +
metrics.latency.granularity
+ "operator" + Defines the granularity of latency metrics. Accepted values are:
  • single - Track latency without differentiating between sources and subtasks.
  • operator - Track latency while differentiating between sources, but not subtasks.
  • subtask - Track latency while differentiating between sources and subtasks.
+
metrics.latency.history-size
128 Defines the number of measured latencies to maintain at each operator. -
metrics.reporter.<name>.<parameter>
+
metrics.latency.interval
+ 0 + Defines the interval at which latency tracking marks are emitted from the sources. Disables latency tracking if set to 0 or a negative value. Enabling this feature can significantly impact the performance of the cluster. + + +
metrics.reporter.<name>.<parameter>
(none) - Configures the parameter <parameter> for the reporter named <name>. + Configures the parameter <parameter> for the reporter named <name>. -
metrics.reporter.<name>.class
+
metrics.reporter.<name>.class
(none) - The reporter class to use for the reporter named <name>. + The reporter class to use for the reporter named <name>. -
metrics.reporter.<name>.interval
+
metrics.reporter.<name>.interval
(none) - The reporter interval to use for the reporter named <name>. + The reporter interval to use for the reporter named <name>.
metrics.reporters
(none) - + An optional list of reporter names. If configured, only reporters whose name matches any of the names in the list will be started. Otherwise, all reporters that could be found in the configuration will be started.
metrics.scope.delimiter
"." - + Delimiter used to assemble the metric identifier.
metrics.scope.jm
- "<host>.jobmanager" + "<host>.jobmanager" Defines the scope format string that is applied to all metrics scoped to a JobManager.
metrics.scope.jm.job
- "<host>.jobmanager.<job_name>" + "<host>.jobmanager.<job_name>" Defines the scope format string that is applied to all metrics scoped to a job on a JobManager.
metrics.scope.operator
- "<host>.taskmanager.<tm_id>.<job_name>.<operator_name>.<subtask_index>" + "<host>.taskmanager.<tm_id>.<job_name>.<operator_name>.<subtask_index>" Defines the scope format string that is applied to all metrics scoped to an operator.
metrics.scope.task
- "<host>.taskmanager.<tm_id>.<job_name>.<task_name>.<subtask_index>" + "<host>.taskmanager.<tm_id>.<job_name>.<task_name>.<subtask_index>" Defines the scope format string that is applied to all metrics scoped to a task.
metrics.scope.tm
- "<host>.taskmanager.<tm_id>" + "<host>.taskmanager.<tm_id>" Defines the scope format string that is applied to all metrics scoped to a TaskManager.
metrics.scope.tm.job
- "<host>.taskmanager.<tm_id>.<job_name>" + "<host>.taskmanager.<tm_id>.<job_name>" Defines the scope format string that is applied to all metrics scoped to a job on a TaskManager. + +
metrics.system-resource
+ false + Flag indicating whether Flink should report system resource metrics such as machine's CPU, memory or network usage. + + +
metrics.system-resource-probing-interval
+ 5000 + Interval between probing of system resource metrics specified in milliseconds. Has an effect only when 'metrics.system-resource' is enabled. + diff --git a/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html b/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html new file mode 100644 index 00000000000000..5eee4c698c28eb --- /dev/null +++ b/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
deleteOnShutdown
trueSpecifies whether to delete metrics from the PushGateway on shutdown.
filterLabelValueCharacters
trueSpecifies whether to filter label value characters. If enabled, all characters not matching [a-zA-Z0-9:_] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the Prometheus requirements.
host
(none)The PushGateway server host.
jobName
(none)The job name under which metrics will be pushed
port
-1The PushGateway server port.
randomJobNameSuffix
trueSpecifies whether a random suffix should be appended to the job name.
diff --git a/docs/_includes/generated/queryable_state_configuration.html b/docs/_includes/generated/queryable_state_configuration.html index c457c40e147154..91eaf912a0cee1 100644 --- a/docs/_includes/generated/queryable_state_configuration.html +++ b/docs/_includes/generated/queryable_state_configuration.html @@ -8,37 +8,42 @@ -
query.client.network-threads
+
queryable-state.client.network-threads
0 Number of network (Netty's event loop) Threads for queryable state client. -
query.proxy.network-threads
+
queryable-state.enable
+ false + Option whether the queryable state proxy and server should be enabled where possible and configurable. + + +
queryable-state.proxy.network-threads
0 Number of network (Netty's event loop) Threads for queryable state proxy. -
query.proxy.ports
+
queryable-state.proxy.ports
"9069" The port range of the queryable state proxy. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234". -
query.proxy.query-threads
+
queryable-state.proxy.query-threads
0 Number of query Threads for queryable state proxy. Uses the number of slots if set to 0. -
query.server.network-threads
+
queryable-state.server.network-threads
0 Number of network (Netty's event loop) Threads for queryable state server. -
query.server.ports
+
queryable-state.server.ports
"9067" The port range of the queryable state server. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234". -
query.server.query-threads
+
queryable-state.server.query-threads
0 Number of query Threads for queryable state server. Uses the number of slots if set to 0. diff --git a/docs/_includes/generated/resource_manager_configuration.html b/docs/_includes/generated/resource_manager_configuration.html index 1b82e51b4ef4c6..3448aba8f52337 100644 --- a/docs/_includes/generated/resource_manager_configuration.html +++ b/docs/_includes/generated/resource_manager_configuration.html @@ -20,7 +20,7 @@
local.number-resourcemanager
1 - + The number of resource managers start.
resourcemanager.job.timeout
@@ -32,5 +32,10 @@ 0 Defines the network port to connect to for communication with the resource manager. By default, the port of the JobManager, because the same ActorSystem is used. Its not possible to use this configuration key to define port ranges. + +
resourcemanager.taskmanager-timeout
+ 30000 + The timeout for an idle task manager to be released. + diff --git a/docs/_includes/generated/rest_configuration.html b/docs/_includes/generated/rest_configuration.html index 20fe96138a5302..2c5f539a480a56 100644 --- a/docs/_includes/generated/rest_configuration.html +++ b/docs/_includes/generated/rest_configuration.html @@ -9,22 +9,37 @@
rest.address
- "localhost" - The address that the server binds itself to / the client connects to. + (none) + The address that should be used by clients to connect to the server.
rest.await-leader-timeout
30000 The time in ms that the client waits for the leader address, e.g., Dispatcher or WebMonitorEndpoint + +
rest.bind-address
+ (none) + The address that the server binds itself. + + +
rest.client.max-content-length
+ 104857600 + The maximum content length in bytes that the client will handle. +
rest.connection-timeout
15000 The maximum time in ms for the client to establish a TCP connection. + +
rest.idleness-timeout
+ 300000 + The maximum time in ms for a connection to stay idle before failing. +
rest.port
- 9065 + 8081 The port that the server listens on / the client connects to. @@ -37,5 +52,20 @@ 20 The number of retries the client will attempt if a retryable operations fails. + +
rest.server.max-content-length
+ 104857600 + The maximum content length in bytes that the server will handle. + + +
rest.server.numThreads
+ 4 + The number of threads for the asynchronous processing of requests. + + +
rest.server.thread-priority
+ 5 + Thread priority of the REST server's executor for processing asynchronous requests. Lowering the thread priority will give Flink's main components more CPU time whereas increasing will allocate more time for the REST server's processing. + diff --git a/docs/_includes/generated/rest_dispatcher.html b/docs/_includes/generated/rest_dispatcher.html deleted file mode 100644 index 25bdff11b33687..00000000000000 --- a/docs/_includes/generated/rest_dispatcher.html +++ /dev/null @@ -1,2048 +0,0 @@ - - - - - - - - - - - - - - - - - - - -
/blobserver/port
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:BlobServerPortResponseBody",
-  "properties" : {
-    "port" : {
-      "type" : "integer"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/config
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:DashboardConfiguration",
-  "properties" : {
-    "refreshInterval" : {
-      "type" : "integer"
-    },
-    "timeZoneName" : {
-      "type" : "string"
-    },
-    "timeZoneOffset" : {
-      "type" : "integer"
-    },
-    "flinkVersion" : {
-      "type" : "string"
-    },
-    "flinkRevision" : {
-      "type" : "string"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/jobmanager/config
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "array",
-  "items" : {
-    "type" : "object",
-    "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:ClusterConfigurationInfoEntry",
-    "properties" : {
-      "key" : {
-        "type" : "string"
-      },
-      "value" : {
-        "type" : "string"
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobmanager/metrics
Verb: GETResponse code: 200 OK
description
Query parameters
-
    -
  • get (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:MetricCollectionResponseBody",
-  "properties" : {
-    "metrics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:Metric",
-        "properties" : {
-          "id" : {
-            "type" : "string"
-          },
-          "value" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/jobs
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:JobIdsWithStatusOverview",
-  "properties" : {
-    "jobsWithStatus" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:JobIdsWithStatusOverview:JobIdWithStatus",
-        "properties" : {
-          "jobId" : {
-            "type" : "object",
-            "id" : "urn:jsonschema:org:apache:flink:api:common:JobID",
-            "properties" : {
-              "upperPart" : {
-                "type" : "integer"
-              },
-              "lowerPart" : {
-                "type" : "integer"
-              },
-              "bytes" : {
-                "type" : "array",
-                "items" : {
-                  "type" : "integer"
-                }
-              }
-            }
-          },
-          "jobStatus" : {
-            "type" : "string",
-            "enum" : [ "CREATED", "RUNNING", "FAILING", "FAILED", "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDED", "RECONCILING" ]
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/jobs
Verb: POSTResponse code: 202 Accepted
description
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobSubmitRequestBody",
-  "properties" : {
-    "serializedJobGraph" : {
-      "type" : "array",
-      "items" : {
-        "type" : "integer"
-      }
-    }
-  }
-}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobSubmitResponseBody",
-  "properties" : {
-    "jobUrl" : {
-      "type" : "string"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/jobs/overview
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:MultipleJobsDetails",
-  "properties" : {
-    "jobs" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:JobDetails",
-        "properties" : {
-          "jobId" : {
-            "type" : "object",
-            "id" : "urn:jsonschema:org:apache:flink:api:common:JobID",
-            "properties" : {
-              "upperPart" : {
-                "type" : "integer"
-              },
-              "lowerPart" : {
-                "type" : "integer"
-              },
-              "bytes" : {
-                "type" : "array",
-                "items" : {
-                  "type" : "integer"
-                }
-              }
-            }
-          },
-          "jobName" : {
-            "type" : "string"
-          },
-          "startTime" : {
-            "type" : "integer"
-          },
-          "endTime" : {
-            "type" : "integer"
-          },
-          "duration" : {
-            "type" : "integer"
-          },
-          "status" : {
-            "type" : "string",
-            "enum" : [ "CREATED", "RUNNING", "FAILING", "FAILED", "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDED", "RECONCILING" ]
-          },
-          "lastUpdateTime" : {
-            "type" : "integer"
-          },
-          "tasksPerState" : {
-            "type" : "array",
-            "items" : {
-              "type" : "integer"
-            }
-          },
-          "numTasks" : {
-            "type" : "integer"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid
Verb: PATCHResponse code: 202 Accepted
description
Path parameters
-
    -
  • jobid - description
    • -
-
Query parameters
-
    -
  • mode (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/accumulators
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/checkpoints
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics",
-  "properties" : {
-    "counts" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:Counts",
-      "properties" : {
-        "numberRestoredCheckpoints" : {
-          "type" : "integer"
-        },
-        "totalNumberCheckpoints" : {
-          "type" : "integer"
-        },
-        "numberInProgressCheckpoints" : {
-          "type" : "integer"
-        },
-        "numberCompletedCheckpoints" : {
-          "type" : "integer"
-        },
-        "numberFailedCheckpoints" : {
-          "type" : "integer"
-        }
-      }
-    },
-    "summary" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:Summary",
-      "properties" : {
-        "stateSize" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
-          "properties" : {
-            "minimum" : {
-              "type" : "integer"
-            },
-            "maximum" : {
-              "type" : "integer"
-            },
-            "average" : {
-              "type" : "integer"
-            }
-          }
-        },
-        "duration" : {
-          "type" : "object",
-          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-        },
-        "alignmentBuffered" : {
-          "type" : "object",
-          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-        }
-      }
-    },
-    "latestCheckpoints" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:LatestCheckpoints",
-      "properties" : {
-        "completedCheckpointStatistics" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:CompletedCheckpointStatistics",
-          "properties" : {
-            "id" : {
-              "type" : "integer"
-            },
-            "status" : {
-              "type" : "string",
-              "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-            },
-            "savepoint" : {
-              "type" : "boolean"
-            },
-            "triggerTimestamp" : {
-              "type" : "integer"
-            },
-            "latestAckTimestamp" : {
-              "type" : "integer"
-            },
-            "stateSize" : {
-              "type" : "integer"
-            },
-            "duration" : {
-              "type" : "integer"
-            },
-            "alignmentBuffered" : {
-              "type" : "integer"
-            },
-            "numSubtasks" : {
-              "type" : "integer"
-            },
-            "numAckSubtasks" : {
-              "type" : "integer"
-            },
-            "checkpointStatisticsPerTask" : {
-              "type" : "object",
-              "additionalProperties" : {
-                "type" : "object",
-                "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
-                "properties" : {
-                  "checkpointId" : {
-                    "type" : "integer"
-                  },
-                  "checkpointStatus" : {
-                    "type" : "string",
-                    "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-                  },
-                  "latestAckTimestamp" : {
-                    "type" : "integer"
-                  },
-                  "stateSize" : {
-                    "type" : "integer"
-                  },
-                  "duration" : {
-                    "type" : "integer"
-                  },
-                  "alignmentBuffered" : {
-                    "type" : "integer"
-                  },
-                  "numSubtasks" : {
-                    "type" : "integer"
-                  },
-                  "numAckSubtasks" : {
-                    "type" : "integer"
-                  }
-                }
-              }
-            },
-            "externalPath" : {
-              "type" : "string"
-            },
-            "discarded" : {
-              "type" : "boolean"
-            }
-          }
-        },
-        "savepointStatistics" : {
-          "type" : "object",
-          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:CompletedCheckpointStatistics"
-        },
-        "failedCheckpointStatistics" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:FailedCheckpointStatistics",
-          "properties" : {
-            "id" : {
-              "type" : "integer"
-            },
-            "status" : {
-              "type" : "string",
-              "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-            },
-            "savepoint" : {
-              "type" : "boolean"
-            },
-            "triggerTimestamp" : {
-              "type" : "integer"
-            },
-            "latestAckTimestamp" : {
-              "type" : "integer"
-            },
-            "stateSize" : {
-              "type" : "integer"
-            },
-            "duration" : {
-              "type" : "integer"
-            },
-            "alignmentBuffered" : {
-              "type" : "integer"
-            },
-            "numSubtasks" : {
-              "type" : "integer"
-            },
-            "numAckSubtasks" : {
-              "type" : "integer"
-            },
-            "checkpointStatisticsPerTask" : {
-              "type" : "object",
-              "additionalProperties" : {
-                "type" : "object",
-                "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics"
-              }
-            },
-            "failureTimestamp" : {
-              "type" : "integer"
-            },
-            "failureMessage" : {
-              "type" : "string"
-            }
-          }
-        },
-        "restoredCheckpointStatistics" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:RestoredCheckpointStatistics",
-          "properties" : {
-            "id" : {
-              "type" : "integer"
-            },
-            "restoreTimestamp" : {
-              "type" : "integer"
-            },
-            "savepoint" : {
-              "type" : "boolean"
-            },
-            "externalPath" : {
-              "type" : "string"
-            }
-          }
-        }
-      }
-    },
-    "history" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
-        "properties" : {
-          "id" : {
-            "type" : "integer"
-          },
-          "status" : {
-            "type" : "string",
-            "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-          },
-          "savepoint" : {
-            "type" : "boolean"
-          },
-          "triggerTimestamp" : {
-            "type" : "integer"
-          },
-          "latestAckTimestamp" : {
-            "type" : "integer"
-          },
-          "stateSize" : {
-            "type" : "integer"
-          },
-          "duration" : {
-            "type" : "integer"
-          },
-          "alignmentBuffered" : {
-            "type" : "integer"
-          },
-          "numSubtasks" : {
-            "type" : "integer"
-          },
-          "numAckSubtasks" : {
-            "type" : "integer"
-          },
-          "checkpointStatisticsPerTask" : {
-            "type" : "object",
-            "additionalProperties" : {
-              "type" : "object",
-              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics"
-            }
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/checkpoints/config
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/checkpoints/details/:checkpointid/subtasks/:vertexid
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • checkpointid - description
    • -
  • vertexid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails",
-  "properties" : {
-    "checkpointId" : {
-      "type" : "integer"
-    },
-    "checkpointStatus" : {
-      "type" : "string",
-      "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-    },
-    "latestAckTimestamp" : {
-      "type" : "integer"
-    },
-    "stateSize" : {
-      "type" : "integer"
-    },
-    "duration" : {
-      "type" : "integer"
-    },
-    "alignmentBuffered" : {
-      "type" : "integer"
-    },
-    "numSubtasks" : {
-      "type" : "integer"
-    },
-    "numAckSubtasks" : {
-      "type" : "integer"
-    },
-    "summary" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:Summary",
-      "properties" : {
-        "stateSize" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
-          "properties" : {
-            "minimum" : {
-              "type" : "integer"
-            },
-            "maximum" : {
-              "type" : "integer"
-            },
-            "average" : {
-              "type" : "integer"
-            }
-          }
-        },
-        "duration" : {
-          "type" : "object",
-          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-        },
-        "checkpointDuration" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointDuration",
-          "properties" : {
-            "synchronousDuration" : {
-              "type" : "object",
-              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-            },
-            "asynchronousDuration" : {
-              "type" : "object",
-              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-            }
-          }
-        },
-        "checkpointAlignment" : {
-          "type" : "object",
-          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointAlignment",
-          "properties" : {
-            "bufferedData" : {
-              "type" : "object",
-              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-            },
-            "duration" : {
-              "type" : "object",
-              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
-            }
-          }
-        }
-      }
-    },
-    "subtaskCheckpointStatistics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:SubtaskCheckpointStatistics",
-        "properties" : {
-          "index" : {
-            "type" : "integer"
-          },
-          "checkpointStatus" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/checkpoints/:checkpointid
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • checkpointid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
-  "properties" : {
-    "id" : {
-      "type" : "integer"
-    },
-    "status" : {
-      "type" : "string",
-      "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-    },
-    "savepoint" : {
-      "type" : "boolean"
-    },
-    "triggerTimestamp" : {
-      "type" : "integer"
-    },
-    "latestAckTimestamp" : {
-      "type" : "integer"
-    },
-    "stateSize" : {
-      "type" : "integer"
-    },
-    "duration" : {
-      "type" : "integer"
-    },
-    "alignmentBuffered" : {
-      "type" : "integer"
-    },
-    "numSubtasks" : {
-      "type" : "integer"
-    },
-    "numAckSubtasks" : {
-      "type" : "integer"
-    },
-    "checkpointStatisticsPerTask" : {
-      "type" : "object",
-      "additionalProperties" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
-        "properties" : {
-          "checkpointId" : {
-            "type" : "integer"
-          },
-          "checkpointStatus" : {
-            "type" : "string",
-            "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
-          },
-          "latestAckTimestamp" : {
-            "type" : "integer"
-          },
-          "stateSize" : {
-            "type" : "integer"
-          },
-          "duration" : {
-            "type" : "integer"
-          },
-          "alignmentBuffered" : {
-            "type" : "integer"
-          },
-          "numSubtasks" : {
-            "type" : "integer"
-          },
-          "numAckSubtasks" : {
-            "type" : "integer"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/config
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobConfigInfo",
-  "properties" : {
-    "jobId" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:api:common:JobID",
-      "properties" : {
-        "upperPart" : {
-          "type" : "integer"
-        },
-        "lowerPart" : {
-          "type" : "integer"
-        },
-        "bytes" : {
-          "type" : "array",
-          "items" : {
-            "type" : "integer"
-          }
-        }
-      }
-    },
-    "jobName" : {
-      "type" : "string"
-    },
-    "executionConfigInfo" : {
-      "type" : "any"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/exceptions
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/metrics
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
Query parameters
-
    -
  • get (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:MetricCollectionResponseBody",
-  "properties" : {
-    "metrics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:Metric",
-        "properties" : {
-          "id" : {
-            "type" : "string"
-          },
-          "value" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/plan
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobPlanInfo",
-  "properties" : {
-    "jsonPlan" : {
-      "type" : "string"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/vertices/:vertexid/accumulators
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • vertexid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/vertices/:vertexid/metrics
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • vertexid - description
    • -
-
Query parameters
-
    -
  • get (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:MetricCollectionResponseBody",
-  "properties" : {
-    "metrics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:Metric",
-        "properties" : {
-          "id" : {
-            "type" : "string"
-          },
-          "value" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/vertices/:vertexid/subtasks/:subtaskindex/metrics
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • vertexid - description
    • -
  • subtaskindex - description
    • -
-
Query parameters
-
    -
  • get (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:MetricCollectionResponseBody",
-  "properties" : {
-    "metrics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:Metric",
-        "properties" : {
-          "id" : {
-            "type" : "string"
-          },
-          "value" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/jobs/:jobid/vertices/:vertexid/subtasktimes
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • jobid - description
    • -
  • vertexid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "any"
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/overview
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:legacy:messages:ClusterOverviewWithVersion",
-  "properties" : {
-    "numJobsRunningOrPending" : {
-      "type" : "integer"
-    },
-    "numJobsFinished" : {
-      "type" : "integer"
-    },
-    "numJobsCancelled" : {
-      "type" : "integer"
-    },
-    "numJobsFailed" : {
-      "type" : "integer"
-    },
-    "numTaskManagersConnected" : {
-      "type" : "integer"
-    },
-    "numSlotsTotal" : {
-      "type" : "integer"
-    },
-    "numSlotsAvailable" : {
-      "type" : "integer"
-    },
-    "version" : {
-      "type" : "string"
-    },
-    "commitId" : {
-      "type" : "string"
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - -
/taskmanagers
Verb: GETResponse code: 200 OK
description
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagersInfo",
-  "properties" : {
-    "taskManagerInfos" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerInfo",
-        "properties" : {
-          "resourceId" : {
-            "type" : "object",
-            "id" : "urn:jsonschema:org:apache:flink:runtime:clusterframework:types:ResourceID",
-            "properties" : {
-              "resourceIdString" : {
-                "type" : "string"
-              },
-              "resourceID" : {
-                "type" : "object",
-                "$ref" : "urn:jsonschema:org:apache:flink:runtime:clusterframework:types:ResourceID"
-              }
-            }
-          },
-          "address" : {
-            "type" : "string"
-          },
-          "dataPort" : {
-            "type" : "integer"
-          },
-          "lastHeartbeat" : {
-            "type" : "integer"
-          },
-          "numberSlots" : {
-            "type" : "integer"
-          },
-          "numberAvailableSlots" : {
-            "type" : "integer"
-          },
-          "hardwareDescription" : {
-            "type" : "object",
-            "id" : "urn:jsonschema:org:apache:flink:runtime:instance:HardwareDescription",
-            "properties" : {
-              "numberOfCPUCores" : {
-                "type" : "integer"
-              },
-              "sizeOfPhysicalMemory" : {
-                "type" : "integer"
-              },
-              "sizeOfJvmHeap" : {
-                "type" : "integer"
-              },
-              "sizeOfManagedMemory" : {
-                "type" : "integer"
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
/taskmanagers/:taskmanagerid
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • taskmanagerid - description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerDetailsInfo",
-  "properties" : {
-    "resourceId" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:clusterframework:types:ResourceID",
-      "properties" : {
-        "resourceIdString" : {
-          "type" : "string"
-        },
-        "resourceID" : {
-          "type" : "object",
-          "$ref" : "urn:jsonschema:org:apache:flink:runtime:clusterframework:types:ResourceID"
-        }
-      }
-    },
-    "address" : {
-      "type" : "string"
-    },
-    "dataPort" : {
-      "type" : "integer"
-    },
-    "lastHeartbeat" : {
-      "type" : "integer"
-    },
-    "numberSlots" : {
-      "type" : "integer"
-    },
-    "numberAvailableSlots" : {
-      "type" : "integer"
-    },
-    "hardwareDescription" : {
-      "type" : "object",
-      "id" : "urn:jsonschema:org:apache:flink:runtime:instance:HardwareDescription",
-      "properties" : {
-        "numberOfCPUCores" : {
-          "type" : "integer"
-        },
-        "sizeOfPhysicalMemory" : {
-          "type" : "integer"
-        },
-        "sizeOfJvmHeap" : {
-          "type" : "integer"
-        },
-        "sizeOfManagedMemory" : {
-          "type" : "integer"
-        }
-      }
-    }
-  }
-}            
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/taskmanagers/:taskmanagerid/metrics
Verb: GETResponse code: 200 OK
description
Path parameters
-
    -
  • taskmanagerid - description
    • -
-
Query parameters
-
    -
  • get (optional): description
    • -
-
- -
- 
-            
-{}            
-
-
-
- -
- 
-            
-{
-  "type" : "object",
-  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:MetricCollectionResponseBody",
-  "properties" : {
-    "metrics" : {
-      "type" : "array",
-      "items" : {
-        "type" : "object",
-        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:Metric",
-        "properties" : {
-          "id" : {
-            "type" : "string"
-          },
-          "value" : {
-            "type" : "string"
-          }
-        }
-      }
-    }
-  }
-}            
-
-
-
diff --git a/docs/_includes/generated/rest_v1_dispatcher.html b/docs/_includes/generated/rest_v1_dispatcher.html new file mode 100644 index 00000000000000..2fd3dcfe40c9a9 --- /dev/null +++ b/docs/_includes/generated/rest_v1_dispatcher.html @@ -0,0 +1,3875 @@ + + + + + + + + + + + + + + + + + + + +
/cluster
Verb: DELETEResponse code: 200 OK
Shuts down the cluster
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/config
Verb: GETResponse code: 200 OK
Returns the configuration of the WebUI.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:DashboardConfiguration",
+  "properties" : {
+    "refresh-interval" : {
+      "type" : "integer"
+    },
+    "timezone-name" : {
+      "type" : "string"
+    },
+    "timezone-offset" : {
+      "type" : "integer"
+    },
+    "flink-version" : {
+      "type" : "string"
+    },
+    "flink-revision" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jars
Verb: GETResponse code: 200 OK
Returns a list of all jars previously uploaded via '/jars/upload'.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo",
+  "properties" : {
+    "address" : {
+      "type" : "string"
+    },
+    "files" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo:JarFileInfo",
+        "properties" : {
+          "id" : {
+            "type" : "string"
+          },
+          "name" : {
+            "type" : "string"
+          },
+          "uploaded" : {
+            "type" : "integer"
+          },
+          "entry" : {
+            "type" : "array",
+            "items" : {
+              "type" : "object",
+              "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo:JarEntryInfo",
+              "properties" : {
+                "name" : {
+                  "type" : "string"
+                },
+                "description" : {
+                  "type" : "string"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jars/upload
Verb: POSTResponse code: 200 OK
Uploads a jar to the cluster. The jar must be sent as multi-part data. Make sure that the "Content-Type" header is set to "application/x-java-archive", as some http libraries do not add the header by default. +Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@path/to/flink-job.jar" http://hostname:port/jars/upload'.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarUploadResponseBody",
+  "properties" : {
+    "filename" : {
+      "type" : "string"
+    },
+    "status" : {
+      "type" : "string",
+      "enum" : [ "success" ]
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jars/:jarid
Verb: DELETEResponse code: 200 OK
Deletes a jar previously uploaded via '/jars/upload'.
Path parameters
+
    +
  • jarid - String value that identifies a jar. When uploading the jar a path is returned, where the filename is the ID. This value is equivalent to the `id` field in the list of uploaded jars (/jars).
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jars/:jarid/plan
Verb: GETResponse code: 200 OK
Returns the dataflow plan of a job contained in a jar previously uploaded via '/jars/upload'. Program arguments can be passed both via the JSON request (recommended) or query parameters.
Path parameters
+
    +
  • jarid - String value that identifies a jar. When uploading the jar a path is returned, where the filename is the ID. This value is equivalent to the `id` field in the list of uploaded jars (/jars).
    • +
+
Query parameters
+
    +
  • program-args (optional): Deprecated, please use 'programArg' instead. String value that specifies the arguments for the program or plan
    • +
  • programArg (optional): Comma-separated list of program arguments.
    • +
  • entry-class (optional): String value that specifies the fully qualified name of the entry point class. Overrides the class defined in the jar file manifest.
    • +
  • parallelism (optional): Positive integer value that specifies the desired parallelism for the job.
    • +
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarPlanRequestBody",
+  "properties" : {
+    "entryClass" : {
+      "type" : "string"
+    },
+    "programArgs" : {
+      "type" : "string"
+    },
+    "programArgsList" : {
+      "type" : "array",
+      "items" : {
+        "type" : "string"
+      }
+    },
+    "parallelism" : {
+      "type" : "integer"
+    }
+  }
+}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobPlanInfo",
+  "properties" : {
+    "plan" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jars/:jarid/run
Verb: POSTResponse code: 200 OK
Submits a job by running a jar previously uploaded via '/jars/upload'. Program arguments can be passed both via the JSON request (recommended) or query parameters.
Path parameters
+
    +
  • jarid - String value that identifies a jar. When uploading the jar a path is returned, where the filename is the ID. This value is equivalent to the `id` field in the list of uploaded jars (/jars).
    • +
+
Query parameters
+
    +
  • allowNonRestoredState (optional): Boolean value that specifies whether the job submission should be rejected if the savepoint contains state that cannot be mapped back to the job.
    • +
  • savepointPath (optional): String value that specifies the path of the savepoint to restore the job from.
    • +
  • program-args (optional): Deprecated, please use 'programArg' instead. String value that specifies the arguments for the program or plan
    • +
  • programArg (optional): Comma-separated list of program arguments.
    • +
  • entry-class (optional): String value that specifies the fully qualified name of the entry point class. Overrides the class defined in the jar file manifest.
    • +
  • parallelism (optional): Positive integer value that specifies the desired parallelism for the job.
    • +
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarRunRequestBody",
+  "properties" : {
+    "entryClass" : {
+      "type" : "string"
+    },
+    "programArgs" : {
+      "type" : "string"
+    },
+    "programArgsList" : {
+      "type" : "array",
+      "items" : {
+        "type" : "string"
+      }
+    },
+    "parallelism" : {
+      "type" : "integer"
+    },
+    "allowNonRestoredState" : {
+      "type" : "boolean"
+    },
+    "savepointPath" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarRunResponseBody",
+  "properties" : {
+    "jobid" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jobmanager/config
Verb: GETResponse code: 200 OK
Returns the cluster configuration.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "array",
+  "items" : {
+    "type" : "object",
+    "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:ClusterConfigurationInfoEntry",
+    "properties" : {
+      "key" : {
+        "type" : "string"
+      },
+      "value" : {
+        "type" : "string"
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobmanager/metrics
Verb: GETResponse code: 200 OK
Provides access to job manager metrics.
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jobs
Verb: GETResponse code: 200 OK
Returns an overview over all jobs and their current state.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:JobIdsWithStatusOverview",
+  "properties" : {
+    "jobs" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:JobIdsWithStatusOverview:JobIdWithStatus",
+        "properties" : {
+          "id" : {
+            "type" : "any"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "CREATED", "RUNNING", "FAILING", "FAILED", "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDING", "SUSPENDED", "RECONCILING" ]
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jobs
Verb: POSTResponse code: 202 Accepted
Submits a job. This call is primarily intended to be used by the Flink client. This call expects a multipart/form-data request that consists of file uploads for the serialized JobGraph, jars and distributed cache artifacts and an attribute named "request" for the JSON payload.
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobSubmitRequestBody",
+  "properties" : {
+    "jobGraphFileName" : {
+      "type" : "string"
+    },
+    "jobJarFileNames" : {
+      "type" : "array",
+      "items" : {
+        "type" : "string"
+      }
+    },
+    "jobArtifactFileNames" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobSubmitRequestBody:DistributedCacheFile",
+        "properties" : {
+          "entryName" : {
+            "type" : "string"
+          },
+          "fileName" : {
+            "type" : "string"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobSubmitResponseBody",
+  "properties" : {
+    "jobUrl" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/metrics
Verb: GETResponse code: 200 OK
Provides access to aggregated job metrics.
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
  • agg (optional): Comma-separated list of aggregation modes which should be calculated. Available aggregations are: "min, max, sum, avg".
    • +
  • jobs (optional): Comma-separated list of 32-character hexadecimal strings to select specific jobs.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/jobs/overview
Verb: GETResponse code: 200 OK
Returns an overview over all jobs.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:messages:webmonitor:MultipleJobsDetails",
+  "properties" : {
+    "jobs" : {
+      "type" : "array",
+      "items" : {
+        "type" : "any"
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid
Verb: GETResponse code: 200 OK
Returns details of a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobDetailsInfo",
+  "properties" : {
+    "jid" : {
+      "type" : "any"
+    },
+    "name" : {
+      "type" : "string"
+    },
+    "isStoppable" : {
+      "type" : "boolean"
+    },
+    "state" : {
+      "type" : "string",
+      "enum" : [ "CREATED", "RUNNING", "FAILING", "FAILED", "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDING", "SUSPENDED", "RECONCILING" ]
+    },
+    "start-time" : {
+      "type" : "integer"
+    },
+    "end-time" : {
+      "type" : "integer"
+    },
+    "duration" : {
+      "type" : "integer"
+    },
+    "now" : {
+      "type" : "integer"
+    },
+    "timestamps" : {
+      "type" : "object",
+      "additionalProperties" : {
+        "type" : "integer"
+      }
+    },
+    "vertices" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobDetailsInfo:JobVertexDetailsInfo",
+        "properties" : {
+          "id" : {
+            "type" : "any"
+          },
+          "name" : {
+            "type" : "string"
+          },
+          "parallelism" : {
+            "type" : "integer"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "CREATED", "SCHEDULED", "DEPLOYING", "RUNNING", "FINISHED", "CANCELING", "CANCELED", "FAILED", "RECONCILING" ]
+          },
+          "start-time" : {
+            "type" : "integer"
+          },
+          "end-time" : {
+            "type" : "integer"
+          },
+          "duration" : {
+            "type" : "integer"
+          },
+          "tasks" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "type" : "integer"
+            }
+          },
+          "metrics" : {
+            "type" : "object",
+            "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:IOMetricsInfo",
+            "properties" : {
+              "read-bytes" : {
+                "type" : "integer"
+              },
+              "read-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "write-bytes" : {
+                "type" : "integer"
+              },
+              "write-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "read-records" : {
+                "type" : "integer"
+              },
+              "read-records-complete" : {
+                "type" : "boolean"
+              },
+              "write-records" : {
+                "type" : "integer"
+              },
+              "write-records-complete" : {
+                "type" : "boolean"
+              }
+            }
+          }
+        }
+      }
+    },
+    "status-counts" : {
+      "type" : "object",
+      "additionalProperties" : {
+        "type" : "integer"
+      }
+    },
+    "plan" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid
Verb: PATCHResponse code: 202 Accepted
Terminates a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
Query parameters
+
    +
  • mode (optional): String value that specifies the termination mode. Supported values are: "cancel, stop".
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/accumulators
Verb: GETResponse code: 200 OK
Returns the accumulators for all tasks of a job, aggregated across the respective subtasks.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
Query parameters
+
    +
  • includeSerializedValue (optional): Boolean value that specifies whether serialized user task accumulators should be included in the response.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobAccumulatorsInfo",
+  "properties" : {
+    "job-accumulators" : {
+      "type" : "array",
+      "items" : {
+        "type" : "any"
+      }
+    },
+    "user-task-accumulators" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobAccumulatorsInfo:UserTaskAccumulator",
+        "properties" : {
+          "name" : {
+            "type" : "string"
+          },
+          "type" : {
+            "type" : "string"
+          },
+          "value" : {
+            "type" : "string"
+          }
+        }
+      }
+    },
+    "serialized-user-task-accumulators" : {
+      "type" : "object",
+      "additionalProperties" : {
+        "type" : "any"
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/checkpoints
Verb: GETResponse code: 200 OK
Returns checkpointing statistics for a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics",
+  "properties" : {
+    "counts" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:Counts",
+      "properties" : {
+        "restored" : {
+          "type" : "integer"
+        },
+        "total" : {
+          "type" : "integer"
+        },
+        "in_progress" : {
+          "type" : "integer"
+        },
+        "completed" : {
+          "type" : "integer"
+        },
+        "failed" : {
+          "type" : "integer"
+        }
+      }
+    },
+    "summary" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:Summary",
+      "properties" : {
+        "state_size" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
+          "properties" : {
+            "min" : {
+              "type" : "integer"
+            },
+            "max" : {
+              "type" : "integer"
+            },
+            "avg" : {
+              "type" : "integer"
+            }
+          }
+        },
+        "end_to_end_duration" : {
+          "type" : "object",
+          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+        },
+        "alignment_buffered" : {
+          "type" : "object",
+          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+        }
+      }
+    },
+    "latest" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:LatestCheckpoints",
+      "properties" : {
+        "completed" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:CompletedCheckpointStatistics",
+          "properties" : {
+            "id" : {
+              "type" : "integer"
+            },
+            "status" : {
+              "type" : "string",
+              "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+            },
+            "is_savepoint" : {
+              "type" : "boolean"
+            },
+            "trigger_timestamp" : {
+              "type" : "integer"
+            },
+            "latest_ack_timestamp" : {
+              "type" : "integer"
+            },
+            "state_size" : {
+              "type" : "integer"
+            },
+            "end_to_end_duration" : {
+              "type" : "integer"
+            },
+            "alignment_buffered" : {
+              "type" : "integer"
+            },
+            "num_subtasks" : {
+              "type" : "integer"
+            },
+            "num_acknowledged_subtasks" : {
+              "type" : "integer"
+            },
+            "tasks" : {
+              "type" : "object",
+              "additionalProperties" : {
+                "type" : "object",
+                "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
+                "properties" : {
+                  "id" : {
+                    "type" : "integer"
+                  },
+                  "status" : {
+                    "type" : "string",
+                    "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+                  },
+                  "latest_ack_timestamp" : {
+                    "type" : "integer"
+                  },
+                  "state_size" : {
+                    "type" : "integer"
+                  },
+                  "end_to_end_duration" : {
+                    "type" : "integer"
+                  },
+                  "alignment_buffered" : {
+                    "type" : "integer"
+                  },
+                  "num_subtasks" : {
+                    "type" : "integer"
+                  },
+                  "num_acknowledged_subtasks" : {
+                    "type" : "integer"
+                  }
+                }
+              }
+            },
+            "external_path" : {
+              "type" : "string"
+            },
+            "discarded" : {
+              "type" : "boolean"
+            }
+          }
+        },
+        "savepoint" : {
+          "type" : "object",
+          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:CompletedCheckpointStatistics"
+        },
+        "failed" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics:FailedCheckpointStatistics",
+          "properties" : {
+            "id" : {
+              "type" : "integer"
+            },
+            "status" : {
+              "type" : "string",
+              "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+            },
+            "is_savepoint" : {
+              "type" : "boolean"
+            },
+            "trigger_timestamp" : {
+              "type" : "integer"
+            },
+            "latest_ack_timestamp" : {
+              "type" : "integer"
+            },
+            "state_size" : {
+              "type" : "integer"
+            },
+            "end_to_end_duration" : {
+              "type" : "integer"
+            },
+            "alignment_buffered" : {
+              "type" : "integer"
+            },
+            "num_subtasks" : {
+              "type" : "integer"
+            },
+            "num_acknowledged_subtasks" : {
+              "type" : "integer"
+            },
+            "tasks" : {
+              "type" : "object",
+              "additionalProperties" : {
+                "type" : "object",
+                "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics"
+              }
+            },
+            "failure_timestamp" : {
+              "type" : "integer"
+            },
+            "failure_message" : {
+              "type" : "string"
+            }
+          }
+        },
+        "restored" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointingStatistics:RestoredCheckpointStatistics",
+          "properties" : {
+            "id" : {
+              "type" : "integer"
+            },
+            "restore_timestamp" : {
+              "type" : "integer"
+            },
+            "is_savepoint" : {
+              "type" : "boolean"
+            },
+            "external_path" : {
+              "type" : "string"
+            }
+          }
+        }
+      }
+    },
+    "history" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
+        "properties" : {
+          "id" : {
+            "type" : "integer"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+          },
+          "is_savepoint" : {
+            "type" : "boolean"
+          },
+          "trigger_timestamp" : {
+            "type" : "integer"
+          },
+          "latest_ack_timestamp" : {
+            "type" : "integer"
+          },
+          "state_size" : {
+            "type" : "integer"
+          },
+          "end_to_end_duration" : {
+            "type" : "integer"
+          },
+          "alignment_buffered" : {
+            "type" : "integer"
+          },
+          "num_subtasks" : {
+            "type" : "integer"
+          },
+          "num_acknowledged_subtasks" : {
+            "type" : "integer"
+          },
+          "tasks" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "type" : "object",
+              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics"
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/checkpoints/config
Verb: GETResponse code: 200 OK
Returns the checkpointing configuration.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointConfigInfo",
+  "properties" : {
+    "mode" : {
+      "type" : "any"
+    },
+    "interval" : {
+      "type" : "integer"
+    },
+    "timeout" : {
+      "type" : "integer"
+    },
+    "min_pause" : {
+      "type" : "integer"
+    },
+    "max_concurrent" : {
+      "type" : "integer"
+    },
+    "externalization" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointConfigInfo:ExternalizedCheckpointInfo",
+      "properties" : {
+        "enabled" : {
+          "type" : "boolean"
+        },
+        "delete_on_cancellation" : {
+          "type" : "boolean"
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/checkpoints/details/:checkpointid
Verb: GETResponse code: 200 OK
Returns details for a checkpoint.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • checkpointid - Long value that identifies a checkpoint.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
+  "properties" : {
+    "id" : {
+      "type" : "integer"
+    },
+    "status" : {
+      "type" : "string",
+      "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+    },
+    "is_savepoint" : {
+      "type" : "boolean"
+    },
+    "trigger_timestamp" : {
+      "type" : "integer"
+    },
+    "latest_ack_timestamp" : {
+      "type" : "integer"
+    },
+    "state_size" : {
+      "type" : "integer"
+    },
+    "end_to_end_duration" : {
+      "type" : "integer"
+    },
+    "alignment_buffered" : {
+      "type" : "integer"
+    },
+    "num_subtasks" : {
+      "type" : "integer"
+    },
+    "num_acknowledged_subtasks" : {
+      "type" : "integer"
+    },
+    "tasks" : {
+      "type" : "object",
+      "additionalProperties" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
+        "properties" : {
+          "id" : {
+            "type" : "integer"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+          },
+          "latest_ack_timestamp" : {
+            "type" : "integer"
+          },
+          "state_size" : {
+            "type" : "integer"
+          },
+          "end_to_end_duration" : {
+            "type" : "integer"
+          },
+          "alignment_buffered" : {
+            "type" : "integer"
+          },
+          "num_subtasks" : {
+            "type" : "integer"
+          },
+          "num_acknowledged_subtasks" : {
+            "type" : "integer"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/checkpoints/details/:checkpointid/subtasks/:vertexid
Verb: GETResponse code: 200 OK
Returns checkpoint statistics for a task and its subtasks.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • checkpointid - Long value that identifies a checkpoint.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails",
+  "properties" : {
+    "id" : {
+      "type" : "integer"
+    },
+    "status" : {
+      "type" : "string",
+      "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+    },
+    "latest_ack_timestamp" : {
+      "type" : "integer"
+    },
+    "state_size" : {
+      "type" : "integer"
+    },
+    "end_to_end_duration" : {
+      "type" : "integer"
+    },
+    "alignment_buffered" : {
+      "type" : "integer"
+    },
+    "num_subtasks" : {
+      "type" : "integer"
+    },
+    "num_acknowledged_subtasks" : {
+      "type" : "integer"
+    },
+    "summary" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:Summary",
+      "properties" : {
+        "state_size" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
+          "properties" : {
+            "min" : {
+              "type" : "integer"
+            },
+            "max" : {
+              "type" : "integer"
+            },
+            "avg" : {
+              "type" : "integer"
+            }
+          }
+        },
+        "end_to_end_duration" : {
+          "type" : "object",
+          "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+        },
+        "checkpoint_duration" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointDuration",
+          "properties" : {
+            "sync" : {
+              "type" : "object",
+              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+            },
+            "async" : {
+              "type" : "object",
+              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+            }
+          }
+        },
+        "alignment" : {
+          "type" : "object",
+          "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointAlignment",
+          "properties" : {
+            "buffered" : {
+              "type" : "object",
+              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+            },
+            "duration" : {
+              "type" : "object",
+              "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+            }
+          }
+        }
+      }
+    },
+    "subtasks" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:SubtaskCheckpointStatistics",
+        "properties" : {
+          "index" : {
+            "type" : "integer"
+          },
+          "status" : {
+            "type" : "string"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/config
Verb: GETResponse code: 200 OK
Returns the configuration of a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/exceptions
Verb: GETResponse code: 200 OK
Returns the non-recoverable exceptions that have been observed by the job. The truncated flag defines whether more exceptions occurred, but are not listed, because the response would otherwise get too big.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobExceptionsInfo",
+  "properties" : {
+    "root-exception" : {
+      "type" : "string"
+    },
+    "timestamp" : {
+      "type" : "integer"
+    },
+    "all-exceptions" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobExceptionsInfo:ExecutionExceptionInfo",
+        "properties" : {
+          "exception" : {
+            "type" : "string"
+          },
+          "task" : {
+            "type" : "string"
+          },
+          "location" : {
+            "type" : "string"
+          },
+          "timestamp" : {
+            "type" : "integer"
+          }
+        }
+      }
+    },
+    "truncated" : {
+      "type" : "boolean"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/execution-result
Verb: GETResponse code: 200 OK
Returns the result of a job execution. Gives access to the execution time of the job and to all accumulators created by this job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:JobExecutionResultResponseBody",
+  "properties" : {
+    "status" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:queue:QueueStatus",
+      "required" : true,
+      "properties" : {
+        "id" : {
+          "type" : "string",
+          "required" : true,
+          "enum" : [ "IN_PROGRESS", "COMPLETED" ]
+        }
+      }
+    },
+    "job-execution-result" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/metrics
Verb: GETResponse code: 200 OK
Provides access to job metrics.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/plan
Verb: GETResponse code: 200 OK
Returns the dataflow plan of a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobPlanInfo",
+  "properties" : {
+    "plan" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/rescaling
Verb: PATCHResponse code: 200 OK
Triggers the rescaling of a job.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
Query parameters
+
    +
  • parallelism (mandatory): Positive integer value that specifies the desired parallelism.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:TriggerResponse",
+  "properties" : {
+    "request-id" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/rescaling/:triggerid
Verb: GETResponse code: 200 OK
Returns the status of a rescaling operation.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • triggerid - 32-character hexadecimal string that identifies an asynchronous operation trigger ID. The ID was returned then the operation was triggered.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:AsynchronousOperationResult",
+  "properties" : {
+    "status" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:queue:QueueStatus",
+      "properties" : {
+        "id" : {
+          "type" : "string",
+          "required" : true,
+          "enum" : [ "IN_PROGRESS", "COMPLETED" ]
+        }
+      }
+    },
+    "operation" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/savepoints
Verb: POSTResponse code: 202 Accepted
Triggers a savepoint, and optionally cancels the job afterwards.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:savepoints:SavepointTriggerRequestBody",
+  "properties" : {
+    "target-directory" : {
+      "type" : "string"
+    },
+    "cancel-job" : {
+      "type" : "boolean"
+    }
+  }
+}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:TriggerResponse",
+  "properties" : {
+    "request-id" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/savepoints/:triggerid
Verb: GETResponse code: 200 OK
Returns the status of a savepoint operation.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • triggerid - 32-character hexadecimal string that identifies an asynchronous operation trigger ID. The ID was returned then the operation was triggered.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:AsynchronousOperationResult",
+  "properties" : {
+    "status" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:queue:QueueStatus",
+      "properties" : {
+        "id" : {
+          "type" : "string",
+          "required" : true,
+          "enum" : [ "IN_PROGRESS", "COMPLETED" ]
+        }
+      }
+    },
+    "operation" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid
Verb: GETResponse code: 200 OK
Returns details for a task, with a summary for each of its subtasks.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexDetailsInfo",
+  "properties" : {
+    "id" : {
+      "type" : "any"
+    },
+    "name" : {
+      "type" : "string"
+    },
+    "parallelism" : {
+      "type" : "integer"
+    },
+    "now" : {
+      "type" : "integer"
+    },
+    "subtasks" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexDetailsInfo:VertexTaskDetail",
+        "properties" : {
+          "subtask" : {
+            "type" : "integer"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "CREATED", "SCHEDULED", "DEPLOYING", "RUNNING", "FINISHED", "CANCELING", "CANCELED", "FAILED", "RECONCILING" ]
+          },
+          "attempt" : {
+            "type" : "integer"
+          },
+          "host" : {
+            "type" : "string"
+          },
+          "start_time" : {
+            "type" : "integer"
+          },
+          "end-time" : {
+            "type" : "integer"
+          },
+          "duration" : {
+            "type" : "integer"
+          },
+          "metrics" : {
+            "type" : "object",
+            "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:IOMetricsInfo",
+            "properties" : {
+              "read-bytes" : {
+                "type" : "integer"
+              },
+              "read-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "write-bytes" : {
+                "type" : "integer"
+              },
+              "write-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "read-records" : {
+                "type" : "integer"
+              },
+              "read-records-complete" : {
+                "type" : "boolean"
+              },
+              "write-records" : {
+                "type" : "integer"
+              },
+              "write-records-complete" : {
+                "type" : "boolean"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/accumulators
Verb: GETResponse code: 200 OK
Returns user-defined accumulators of a task, aggregated across all subtasks.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexAccumulatorsInfo",
+  "properties" : {
+    "id" : {
+      "type" : "string"
+    },
+    "user-accumulators" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:UserAccumulator",
+        "properties" : {
+          "name" : {
+            "type" : "string"
+          },
+          "type" : {
+            "type" : "string"
+          },
+          "value" : {
+            "type" : "string"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/backpressure
Verb: GETResponse code: 200 OK
Returns back-pressure information for a job, and may initiate back-pressure sampling if necessary.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexBackPressureInfo",
+  "properties" : {
+    "status" : {
+      "type" : "string",
+      "enum" : [ "deprecated", "ok" ]
+    },
+    "backpressure-level" : {
+      "type" : "string",
+      "enum" : [ "ok", "low", "high" ]
+    },
+    "end-timestamp" : {
+      "type" : "integer"
+    },
+    "subtasks" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexBackPressureInfo:SubtaskBackPressureInfo",
+        "properties" : {
+          "subtask" : {
+            "type" : "integer"
+          },
+          "backpressure-level" : {
+            "type" : "string",
+            "enum" : [ "ok", "low", "high" ]
+          },
+          "ratio" : {
+            "type" : "number"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/metrics
Verb: GETResponse code: 200 OK
Provides access to task metrics.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/accumulators
Verb: GETResponse code: 200 OK
Returns all user-defined accumulators for all subtasks of a task.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtasksAllAccumulatorsInfo",
+  "properties" : {
+    "id" : {
+      "type" : "any"
+    },
+    "parallelism" : {
+      "type" : "integer"
+    },
+    "subtasks" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtasksAllAccumulatorsInfo:SubtaskAccumulatorsInfo",
+        "properties" : {
+          "subtask" : {
+            "type" : "integer"
+          },
+          "attempt" : {
+            "type" : "integer"
+          },
+          "host" : {
+            "type" : "string"
+          },
+          "user-accumulators" : {
+            "type" : "array",
+            "items" : {
+              "type" : "object",
+              "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:UserAccumulator",
+              "properties" : {
+                "name" : {
+                  "type" : "string"
+                },
+                "type" : {
+                  "type" : "string"
+                },
+                "value" : {
+                  "type" : "string"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/metrics
Verb: GETResponse code: 200 OK
Provides access to aggregated subtask metrics.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
  • agg (optional): Comma-separated list of aggregation modes which should be calculated. Available aggregations are: "min, max, sum, avg".
    • +
  • subtasks (optional): Comma-separated list of integer ranges (e.g. "1,3,5-9") to select specific subtasks.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/:subtaskindex
Verb: GETResponse code: 200 OK
Returns details of the current or latest execution attempt of a subtask.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
  • subtaskindex - Positive integer value that identifies a subtask.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtaskExecutionAttemptDetailsInfo",
+  "properties" : {
+    "subtask" : {
+      "type" : "integer"
+    },
+    "status" : {
+      "type" : "string",
+      "enum" : [ "CREATED", "SCHEDULED", "DEPLOYING", "RUNNING", "FINISHED", "CANCELING", "CANCELED", "FAILED", "RECONCILING" ]
+    },
+    "attempt" : {
+      "type" : "integer"
+    },
+    "host" : {
+      "type" : "string"
+    },
+    "start-time" : {
+      "type" : "integer"
+    },
+    "end-time" : {
+      "type" : "integer"
+    },
+    "duration" : {
+      "type" : "integer"
+    },
+    "metrics" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:IOMetricsInfo",
+      "properties" : {
+        "read-bytes" : {
+          "type" : "integer"
+        },
+        "read-bytes-complete" : {
+          "type" : "boolean"
+        },
+        "write-bytes" : {
+          "type" : "integer"
+        },
+        "write-bytes-complete" : {
+          "type" : "boolean"
+        },
+        "read-records" : {
+          "type" : "integer"
+        },
+        "read-records-complete" : {
+          "type" : "boolean"
+        },
+        "write-records" : {
+          "type" : "integer"
+        },
+        "write-records-complete" : {
+          "type" : "boolean"
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/:subtaskindex/attempts/:attempt
Verb: GETResponse code: 200 OK
Returns details of an execution attempt of a subtask. Multiple execution attempts happen in case of failure/recovery.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
  • subtaskindex - Positive integer value that identifies a subtask.
    • +
  • attempt - Positive integer value that identifies an execution attempt.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtaskExecutionAttemptDetailsInfo",
+  "properties" : {
+    "subtask" : {
+      "type" : "integer"
+    },
+    "status" : {
+      "type" : "string",
+      "enum" : [ "CREATED", "SCHEDULED", "DEPLOYING", "RUNNING", "FINISHED", "CANCELING", "CANCELED", "FAILED", "RECONCILING" ]
+    },
+    "attempt" : {
+      "type" : "integer"
+    },
+    "host" : {
+      "type" : "string"
+    },
+    "start-time" : {
+      "type" : "integer"
+    },
+    "end-time" : {
+      "type" : "integer"
+    },
+    "duration" : {
+      "type" : "integer"
+    },
+    "metrics" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:IOMetricsInfo",
+      "properties" : {
+        "read-bytes" : {
+          "type" : "integer"
+        },
+        "read-bytes-complete" : {
+          "type" : "boolean"
+        },
+        "write-bytes" : {
+          "type" : "integer"
+        },
+        "write-bytes-complete" : {
+          "type" : "boolean"
+        },
+        "read-records" : {
+          "type" : "integer"
+        },
+        "read-records-complete" : {
+          "type" : "boolean"
+        },
+        "write-records" : {
+          "type" : "integer"
+        },
+        "write-records-complete" : {
+          "type" : "boolean"
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/:subtaskindex/attempts/:attempt/accumulators
Verb: GETResponse code: 200 OK
Returns the accumulators of an execution attempt of a subtask. Multiple execution attempts happen in case of failure/recovery.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
  • subtaskindex - Positive integer value that identifies a subtask.
    • +
  • attempt - Positive integer value that identifies an execution attempt.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtaskExecutionAttemptAccumulatorsInfo",
+  "properties" : {
+    "subtask" : {
+      "type" : "integer"
+    },
+    "attempt" : {
+      "type" : "integer"
+    },
+    "id" : {
+      "type" : "string"
+    },
+    "user-accumulators" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:UserAccumulator",
+        "properties" : {
+          "name" : {
+            "type" : "string"
+          },
+          "type" : {
+            "type" : "string"
+          },
+          "value" : {
+            "type" : "string"
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasks/:subtaskindex/metrics
Verb: GETResponse code: 200 OK
Provides access to subtask metrics.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
  • subtaskindex - Positive integer value that identifies a subtask.
    • +
+
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/subtasktimes
Verb: GETResponse code: 200 OK
Returns time-related information for all subtasks of a task.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:SubtasksTimesInfo",
+  "properties" : {
+    "id" : {
+      "type" : "string"
+    },
+    "name" : {
+      "type" : "string"
+    },
+    "now" : {
+      "type" : "integer"
+    },
+    "subtasks" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:SubtasksTimesInfo:SubtaskTimeInfo",
+        "properties" : {
+          "subtask" : {
+            "type" : "integer"
+          },
+          "host" : {
+            "type" : "string"
+          },
+          "duration" : {
+            "type" : "integer"
+          },
+          "timestamps" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "type" : "integer"
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/jobs/:jobid/vertices/:vertexid/taskmanagers
Verb: GETResponse code: 200 OK
Returns task information aggregated by task manager.
Path parameters
+
    +
  • jobid - 32-character hexadecimal string value that identifies a job.
    • +
  • vertexid - 32-character hexadecimal string value that identifies a job vertex.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexTaskManagersInfo",
+  "properties" : {
+    "id" : {
+      "type" : "any"
+    },
+    "name" : {
+      "type" : "string"
+    },
+    "now" : {
+      "type" : "integer"
+    },
+    "taskmanagers" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexTaskManagersInfo:TaskManagersInfo",
+        "properties" : {
+          "host" : {
+            "type" : "string"
+          },
+          "status" : {
+            "type" : "string",
+            "enum" : [ "CREATED", "SCHEDULED", "DEPLOYING", "RUNNING", "FINISHED", "CANCELING", "CANCELED", "FAILED", "RECONCILING" ]
+          },
+          "start-time" : {
+            "type" : "integer"
+          },
+          "end-time" : {
+            "type" : "integer"
+          },
+          "duration" : {
+            "type" : "integer"
+          },
+          "metrics" : {
+            "type" : "object",
+            "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:metrics:IOMetricsInfo",
+            "properties" : {
+              "read-bytes" : {
+                "type" : "integer"
+              },
+              "read-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "write-bytes" : {
+                "type" : "integer"
+              },
+              "write-bytes-complete" : {
+                "type" : "boolean"
+              },
+              "read-records" : {
+                "type" : "integer"
+              },
+              "read-records-complete" : {
+                "type" : "boolean"
+              },
+              "write-records" : {
+                "type" : "integer"
+              },
+              "write-records-complete" : {
+                "type" : "boolean"
+              }
+            }
+          },
+          "status-counts" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "type" : "integer"
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/overview
Verb: GETResponse code: 200 OK
Returns an overview over the Flink cluster.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:legacy:messages:ClusterOverviewWithVersion",
+  "properties" : {
+    "taskmanagers" : {
+      "type" : "integer"
+    },
+    "slots-total" : {
+      "type" : "integer"
+    },
+    "slots-available" : {
+      "type" : "integer"
+    },
+    "jobs-running" : {
+      "type" : "integer"
+    },
+    "jobs-finished" : {
+      "type" : "integer"
+    },
+    "jobs-cancelled" : {
+      "type" : "integer"
+    },
+    "jobs-failed" : {
+      "type" : "integer"
+    },
+    "flink-version" : {
+      "type" : "string"
+    },
+    "flink-commit" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/savepoint-disposal
Verb: POSTResponse code: 200 OK
Triggers the desposal of a savepoint.
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:savepoints:SavepointDisposalRequest",
+  "properties" : {
+    "savepoint-path" : {
+      "type" : "string"
+    }
+  }
+}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:TriggerResponse",
+  "properties" : {
+    "request-id" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/savepoint-disposal/:triggerid
Verb: GETResponse code: 200 OK
Returns the status of a savepoint disposal operation.
Path parameters
+
    +
  • triggerid - 32-character hexadecimal string that identifies an asynchronous operation trigger ID. The ID was returned then the operation was triggered.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:handler:async:AsynchronousOperationResult",
+  "properties" : {
+    "status" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:queue:QueueStatus",
+      "properties" : {
+        "id" : {
+          "type" : "string",
+          "required" : true,
+          "enum" : [ "IN_PROGRESS", "COMPLETED" ]
+        }
+      }
+    },
+    "operation" : {
+      "type" : "any"
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + +
/taskmanagers
Verb: GETResponse code: 200 OK
Returns an overview over all task managers.
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagersInfo",
+  "properties" : {
+    "taskmanagers" : {
+      "type" : "array",
+      "items" : {
+        "type" : "object",
+        "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerInfo",
+        "properties" : {
+          "id" : {
+            "type" : "any"
+          },
+          "path" : {
+            "type" : "string"
+          },
+          "dataPort" : {
+            "type" : "integer"
+          },
+          "timeSinceLastHeartbeat" : {
+            "type" : "integer"
+          },
+          "slotsNumber" : {
+            "type" : "integer"
+          },
+          "freeSlots" : {
+            "type" : "integer"
+          },
+          "hardware" : {
+            "type" : "object",
+            "id" : "urn:jsonschema:org:apache:flink:runtime:instance:HardwareDescription",
+            "properties" : {
+              "cpuCores" : {
+                "type" : "integer"
+              },
+              "physicalMemory" : {
+                "type" : "integer"
+              },
+              "freeMemory" : {
+                "type" : "integer"
+              },
+              "managedMemory" : {
+                "type" : "integer"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/taskmanagers/metrics
Verb: GETResponse code: 200 OK
Provides access to aggregated task manager metrics.
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
  • agg (optional): Comma-separated list of aggregation modes which should be calculated. Available aggregations are: "min, max, sum, avg".
    • +
  • taskmanagers (optional): Comma-separated list of 32-character hexadecimal strings to select specific task managers.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
/taskmanagers/:taskmanagerid
Verb: GETResponse code: 200 OK
Returns details for a task manager.
Path parameters
+
    +
  • taskmanagerid - 32-character hexadecimal string that identifies a task manager.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "object",
+  "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerDetailsInfo",
+  "properties" : {
+    "id" : {
+      "type" : "any"
+    },
+    "path" : {
+      "type" : "string"
+    },
+    "dataPort" : {
+      "type" : "integer"
+    },
+    "timeSinceLastHeartbeat" : {
+      "type" : "integer"
+    },
+    "slotsNumber" : {
+      "type" : "integer"
+    },
+    "freeSlots" : {
+      "type" : "integer"
+    },
+    "hardware" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:instance:HardwareDescription",
+      "properties" : {
+        "cpuCores" : {
+          "type" : "integer"
+        },
+        "physicalMemory" : {
+          "type" : "integer"
+        },
+        "freeMemory" : {
+          "type" : "integer"
+        },
+        "managedMemory" : {
+          "type" : "integer"
+        }
+      }
+    },
+    "metrics" : {
+      "type" : "object",
+      "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerMetricsInfo",
+      "properties" : {
+        "heapUsed" : {
+          "type" : "integer"
+        },
+        "heapCommitted" : {
+          "type" : "integer"
+        },
+        "heapMax" : {
+          "type" : "integer"
+        },
+        "nonHeapUsed" : {
+          "type" : "integer"
+        },
+        "nonHeapCommitted" : {
+          "type" : "integer"
+        },
+        "nonHeapMax" : {
+          "type" : "integer"
+        },
+        "directCount" : {
+          "type" : "integer"
+        },
+        "directUsed" : {
+          "type" : "integer"
+        },
+        "directMax" : {
+          "type" : "integer"
+        },
+        "mappedCount" : {
+          "type" : "integer"
+        },
+        "mappedUsed" : {
+          "type" : "integer"
+        },
+        "mappedMax" : {
+          "type" : "integer"
+        },
+        "memorySegmentsAvailable" : {
+          "type" : "integer"
+        },
+        "memorySegmentsTotal" : {
+          "type" : "integer"
+        },
+        "garbageCollectors" : {
+          "type" : "array",
+          "items" : {
+            "type" : "object",
+            "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:taskmanager:TaskManagerMetricsInfo:GarbageCollectorInfo",
+            "properties" : {
+              "name" : {
+                "type" : "string"
+              },
+              "count" : {
+                "type" : "integer"
+              },
+              "time" : {
+                "type" : "integer"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}            
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
/taskmanagers/:taskmanagerid/metrics
Verb: GETResponse code: 200 OK
Provides access to task manager metrics.
Path parameters
+
    +
  • taskmanagerid - 32-character hexadecimal string that identifies a task manager.
    • +
+
Query parameters
+
    +
  • get (optional): Comma-separated list of string values to select specific metrics.
    • +
+
+ +
+ 
+            
+{}            
+
+
+
+ +
+ 
+            
+{
+  "type" : "any"
+}            
+
+
+
+ +{% top %} diff --git a/docs/_includes/generated/rocks_db_configuration.html b/docs/_includes/generated/rocks_db_configuration.html new file mode 100644 index 00000000000000..81f6b53f11793e --- /dev/null +++ b/docs/_includes/generated/rocks_db_configuration.html @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
state.backend.rocksdb.checkpoint.restore.thread.num
1The number of threads used to download files from DFS in RocksDBStateBackend.
state.backend.rocksdb.localdir
(none)The local directory (on the TaskManager) where RocksDB puts its files.
state.backend.rocksdb.timer-service.factory
"HEAP"This determines the factory for timer service state implementation. Options are either HEAP (heap-based, default) or ROCKSDB for an implementation based on RocksDB .
diff --git a/docs/_includes/generated/rocks_db_native_metric_configuration.html b/docs/_includes/generated/rocks_db_native_metric_configuration.html new file mode 100644 index 00000000000000..78fc523b35bf8c --- /dev/null +++ b/docs/_includes/generated/rocks_db_native_metric_configuration.html @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultDescription
state.backend.rocksdb.metrics.actual-delayed-write-rate
falseMonitor the current actual delayed write rate. 0 means no delay.
state.backend.rocksdb.metrics.background-errors
falseMonitor the number of background errors in RocksDB.
state.backend.rocksdb.metrics.compaction-pending
falseTrack pending compactions in RocksDB. Returns 1 if a compaction is pending, 0 otherwise.
state.backend.rocksdb.metrics.cur-size-active-mem-table
falseMonitor the approximate size of the active memtable in bytes.
state.backend.rocksdb.metrics.cur-size-all-mem-tables
falseMonitor the approximate size of the active and unflushed immutable memtables in bytes.
state.backend.rocksdb.metrics.estimate-live-data-size
falseEstimate of the amount of live data in bytes.
state.backend.rocksdb.metrics.estimate-num-keys
falseEstimate the number of keys in RocksDB.
state.backend.rocksdb.metrics.estimate-pending-compaction-bytes
falseEstimated total number of bytes compaction needs to rewrite to get all levels down to under target size. Not valid for other compactions than level-based.
state.backend.rocksdb.metrics.estimate-table-readers-mem
falseEstimate the memory used for reading SST tables, excluding memory used in block cache (e.g.,filter and index blocks) in bytes.
state.backend.rocksdb.metrics.mem-table-flush-pending
falseMonitor the number of pending memtable flushes in RocksDB.
state.backend.rocksdb.metrics.num-deletes-active-mem-table
falseMonitor the total number of delete entries in the active memtable.
state.backend.rocksdb.metrics.num-deletes-imm-mem-tables
falseMonitor the total number of delete entries in the unflushed immutable memtables.
state.backend.rocksdb.metrics.num-entries-active-mem-table
falseMonitor the total number of entries in the active memtable.
state.backend.rocksdb.metrics.num-entries-imm-mem-tables
falseMonitor the total number of entries in the unflushed immutable memtables.
state.backend.rocksdb.metrics.num-immutable-mem-table
falseMonitor the number of immutable memtables in RocksDB.
state.backend.rocksdb.metrics.num-live-versions
falseMonitor number of live versions. Version is an internal data structure. See RocksDB file version_set.h for details. More live versions often mean more SST files are held from being deleted, by iterators or unfinished compactions.
state.backend.rocksdb.metrics.num-running-compactions
falseMonitor the number of currently running compactions.
state.backend.rocksdb.metrics.num-running-flushes
falseMonitor the number of currently running flushes.
state.backend.rocksdb.metrics.num-snapshots
falseMonitor the number of unreleased snapshots of the database.
state.backend.rocksdb.metrics.size-all-mem-tables
falseMonitor the approximate size of the active, unflushed immutable, and pinned immutable memtables in bytes.
state.backend.rocksdb.metrics.total-sst-files-size
falseMonitor the total size (bytes) of all SST files.WARNING: may slow down online queries if there are too many files.
diff --git a/docs/_includes/generated/security_configuration.html b/docs/_includes/generated/security_configuration.html index cd682ecaf0f7eb..8999336926f6d4 100644 --- a/docs/_includes/generated/security_configuration.html +++ b/docs/_includes/generated/security_configuration.html @@ -10,12 +10,57 @@
security.ssl.algorithms
"TLS_RSA_WITH_AES_128_CBC_SHA" - The comma separated list of standard SSL algorithms to be supported. Read more <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#ciphersuites">here</a>. + The comma separated list of standard SSL algorithms to be supported. Read more here -
security.ssl.enabled
+
security.ssl.internal.close-notify-flush-timeout
+ -1 + The timeout (in ms) for flushing the `close_notify` that was triggered by closing a channel. If the `close_notify` was not flushed in the given timeout the channel will be closed forcibly. (-1 = use system default) + + +
security.ssl.internal.enabled
false - Turns on SSL for internal network communication. This can be optionally overridden by flags defined in different transport modules. + Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc). + + +
security.ssl.internal.handshake-timeout
+ -1 + The timeout (in ms) during SSL handshake. (-1 = use system default) + + +
security.ssl.internal.key-password
+ (none) + The secret to decrypt the key in the keystore for Flink's internal endpoints (rpc, data transport, blob server). + + +
security.ssl.internal.keystore
+ (none) + The Java keystore file with SSL Key and Certificate, to be used Flink's internal endpoints (rpc, data transport, blob server). + + +
security.ssl.internal.keystore-password
+ (none) + The secret to decrypt the keystore file for Flink's for Flink's internal endpoints (rpc, data transport, blob server). + + +
security.ssl.internal.session-cache-size
+ -1 + The size of the cache used for storing SSL session objects. According to https://github.com/netty/netty/issues/832, you should always set this to an appropriate number to not run into a bug with stalling IO threads during garbage collection. (-1 = use system default). + + +
security.ssl.internal.session-timeout
+ -1 + The timeout (in ms) for the cached SSL session objects. (-1 = use system default) + + +
security.ssl.internal.truststore
+ (none) + The truststore file containing the public CA certificates to verify the peer for Flink's internal endpoints (rpc, data transport, blob server). + + +
security.ssl.internal.truststore-password
+ (none) + The password to decrypt the truststore for Flink's internal endpoints (rpc, data transport, blob server).
security.ssl.key-password
@@ -37,6 +82,41 @@ "TLSv1.2" The SSL protocol version to be supported for the ssl transport. Note that it doesn’t support comma separated list. + +
security.ssl.rest.authentication-enabled
+ false + Turns on mutual SSL authentication for external communication via the REST endpoints. + + +
security.ssl.rest.enabled
+ false + Turns on SSL for external communication via the REST endpoints. + + +
security.ssl.rest.key-password
+ (none) + The secret to decrypt the key in the keystore for Flink's external REST endpoints. + + +
security.ssl.rest.keystore
+ (none) + The Java keystore file with SSL Key and Certificate, to be used Flink's external REST endpoints. + + +
security.ssl.rest.keystore-password
+ (none) + The secret to decrypt the keystore file for Flink's for Flink's external REST endpoints. + + +
security.ssl.rest.truststore
+ (none) + The truststore file containing the public CA certificates to verify the peer for Flink's external REST endpoints. + + +
security.ssl.rest.truststore-password
+ (none) + The password to decrypt the truststore for Flink's external REST endpoints. +
security.ssl.truststore
(none) diff --git a/docs/_includes/generated/slot_manager_configuration.html b/docs/_includes/generated/slot_manager_configuration.html deleted file mode 100644 index 1517a395f2730a..00000000000000 --- a/docs/_includes/generated/slot_manager_configuration.html +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - -
KeyDefaultDescription
slotmanager.request-timeout
600000The timeout for a slot request to be discarded.
slotmanager.taskmanager-timeout
30000The timeout for an idle task manager to be released.
diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html index bd422445dfb7a5..2093531211d4b1 100644 --- a/docs/_includes/generated/task_manager_configuration.html +++ b/docs/_includes/generated/task_manager_configuration.html @@ -20,7 +20,7 @@
task.cancellation.timers.timeout
7500 - + Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.
task.checkpoint.alignment.max-size
@@ -35,53 +35,38 @@
taskmanager.data.ssl.enabled
true - Enable SSL support for the taskmanager data transport. This is applicable only when the global ssl flag security.ssl.enabled is set to true + Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true -
taskmanager.debug.memory.logIntervalMs
- 5000 - The interval (in ms) for the log thread to log the current memory usage. - - -
taskmanager.debug.memory.startLogThread
+
taskmanager.debug.memory.log
false Flag indicating whether to start a thread, which repeatedly logs the memory usage of the JVM. + +
taskmanager.debug.memory.log-interval
+ 5000 + The interval (in ms) for the log thread to log the current memory usage. +
taskmanager.exit-on-fatal-akka-error
false Whether the quarantine monitor for task managers shall be started. The quarantine monitor shuts down the actor system if it detects that it has quarantined another actor system or if it has been quarantined by another actor system. -
taskmanager.heap.mb
- 1024 - JVM heap size (in megabytes) for the TaskManagers, which are the parallel workers of the system. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value. +
taskmanager.heap.size
+ "1024m" + JVM heap size for the TaskManagers, which are the parallel workers of the system. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value.
taskmanager.host
(none) The hostname of the network interface that the TaskManager binds to. By default, the TaskManager searches for network interfaces that can connect to the JobManager and other TaskManagers. This option can be used to define a hostname if that strategy fails for some reason. Because different TaskManagers need different values for this option, it usually is specified in an additional non-shared TaskManager-specific config file. - -
taskmanager.initial-registration-pause
- "500 ms" - The initial registration pause between two consecutive registration attempts. The pause is doubled for each new registration attempt until it reaches the maximum registration pause. -
taskmanager.jvm-exit-on-oom
false Whether to kill the TaskManager when the task thread throws an OutOfMemoryError. - -
taskmanager.max-registration-pause
- "30 s" - The maximum registration pause between two consecutive registration attempts. The max registration pause requires a time unit specifier (ms/s/min/h/d). - - -
taskmanager.maxRegistrationDuration
- "Inf" - Defines the maximum time it can take for the TaskManager registration. If the duration is exceeded without a successful registration, then the TaskManager terminates. -
taskmanager.memory.fraction
0.7 @@ -99,13 +84,13 @@
taskmanager.memory.segment-size
- 32768 - Size of memory buffers used by the network stack and the memory manager (in bytes). + "32kb" + Size of memory buffers used by the network stack and the memory manager.
taskmanager.memory.size
- -1 - Amount of memory to be allocated by the task manager's memory manager (in megabytes). If not set, a relative fraction will be allocated. + "0" + Amount of memory to be allocated by the task manager's memory manager. If not set, a relative fraction will be allocated.
taskmanager.network.detailed-metrics
@@ -115,12 +100,12 @@
taskmanager.network.memory.buffers-per-channel
2 - Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel). + Maximum number of network buffers to use for each outgoing/incoming channel (subpartition/input channel).In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization.
taskmanager.network.memory.floating-buffers-per-gate
8 - Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). + Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster.
taskmanager.network.memory.fraction
@@ -129,23 +114,23 @@
taskmanager.network.memory.max
- 1073741824 - Maximum memory size for network buffers (in bytes). + "1gb" + Maximum memory size for network buffers.
taskmanager.network.memory.min
- 67108864 - Minimum memory size for network buffers (in bytes). + "64mb" + Minimum memory size for network buffers.
taskmanager.network.request-backoff.initial
100 - Minimum backoff for partition requests of input channels. + Minimum backoff in milliseconds for partition requests of input channels.
taskmanager.network.request-backoff.max
10000 - Maximum backoff for partition requests of input channels. + Maximum backoff in milliseconds for partition requests of input channels.
taskmanager.numberOfTaskSlots
@@ -153,9 +138,24 @@ The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores). -
taskmanager.refused-registration-pause
+
taskmanager.registration.initial-backoff
+ "500 ms" + The initial registration backoff between two consecutive registration attempts. The backoff is doubled for each new registration attempt until it reaches the maximum registration backoff. + + +
taskmanager.registration.max-backoff
+ "30 s" + The maximum registration backoff between two consecutive registration attempts. The max registration backoff requires a time unit specifier (ms/s/min/h/d). + + +
taskmanager.registration.refused-backoff
"10 s" - The pause after a registration has been refused by the job manager before retrying to connect. + The backoff after a registration has been refused by the job manager before retrying to connect. + + +
taskmanager.registration.timeout
+ "5 min" + Defines the timeout for the TaskManager registration. If the duration is exceeded without a successful registration, then the TaskManager terminates.
taskmanager.rpc.port
diff --git a/docs/_includes/generated/web_configuration.html b/docs/_includes/generated/web_configuration.html index 23bf5ab63cb8fc..6fb35e9284ec9a 100644 --- a/docs/_includes/generated/web_configuration.html +++ b/docs/_includes/generated/web_configuration.html @@ -10,82 +10,77 @@
web.access-control-allow-origin
"*" - + Access-Control-Allow-Origin header for all responses from the web-frontend.
web.address
(none) - + Address for runtime monitor web-frontend server.
web.backpressure.cleanup-interval
600000 - + Time, in milliseconds, after which cached stats are cleaned up if not accessed.
web.backpressure.delay-between-samples
50 - + Delay between stack trace samples to determine back pressure in milliseconds.
web.backpressure.num-samples
100 - + Number of stack trace samples to take to determine back pressure.
web.backpressure.refresh-interval
60000 - + Time, in milliseconds, after which available stats are deprecated and need to be refreshed (by resampling).
web.checkpoints.history
10 - + Number of checkpoints to remember for recent history.
web.history
5 - + Number of archived jobs for the JobManager.
web.log.path
(none) - - - -
web.port
- 8081 - + Path to the log file (may be in /log for standalone but under log directory when using YARN).
web.refresh-interval
3000 - + Refresh interval for the web-frontend in milliseconds.
web.ssl.enabled
true - + Flag indicating whether to override SSL support for the JobManager Web UI.
web.submit.enable
true - + Flag indicating whether jobs can be uploaded and run from the web-frontend.
web.timeout
10000 - + Timeout for asynchronous operations by the WebRuntimeMonitor in milliseconds.
web.tmpdir
- (none) - + System.getProperty("java.io.tmpdir") + Flink web directory which is used by the webmonitor.
web.upload.dir
(none) - + Directory for uploading the job jars. If not specified a dynamic directory will be used under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY. diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html index fb16e5340c8f3d..bbe25499f175a4 100644 --- a/docs/_includes/generated/yarn_config_configuration.html +++ b/docs/_includes/generated/yarn_config_configuration.html @@ -30,7 +30,7 @@
yarn.containers.vcores
-1 - The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise. + The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise. In order for this parameter to be used your cluster must have CPU scheduling enabled. You can do this by setting the `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`.
yarn.heartbeat-delay
diff --git a/docs/_includes/sidenav.html b/docs/_includes/sidenav.html index 7afc5f1f398af3..5c5fb51d1260b3 100644 --- a/docs/_includes/sidenav.html +++ b/docs/_includes/sidenav.html @@ -126,7 +126,8 @@ {% endif %} {% endfor %}
  • -
  • Javadocs
    • +
  • Javadocs
    • +
  • Scaladocs
  • Project Page
    • diff --git a/docs/_layouts/base.html b/docs/_layouts/base.html index 1e360cf5dde1d4..1dbaa76696af1e 100644 --- a/docs/_layouts/base.html +++ b/docs/_layouts/base.html @@ -26,6 +26,7 @@ Apache Flink {{ site.version_title }} Documentation: {{ page.title }} + @@ -81,7 +82,7 @@ - + diff --git a/docs/build_docs.sh b/docs/build_docs.sh index f9ffc5a02c9b6e..aa08f4c21f1566 100755 --- a/docs/build_docs.sh +++ b/docs/build_docs.sh @@ -27,16 +27,18 @@ DIR="`pwd`" # We need at least bundler to proceed if [ "`command -v bundle`" == "" ]; then - echo "WARN: Could not find bundle." - echo "Attempting to install locally. If this doesn't work, please install with 'gem install bundler'." + RUBYGEM_BINDIR="" - # Adjust the PATH to discover the locally installed Ruby gem - if which ${RUBY} >/dev/null && which gem >/dev/null; then - export PATH="$(${RUBY} -rubygems -e 'puts Gem.user_dir')/bin:$PATH" - fi + # Adjust the PATH to discover locally installed ruby gem binaries + export PATH="$(${RUBY} -e 'puts Gem.user_dir')/bin:$PATH" - # install bundler locally - ${GEM} install --user-install bundler + if [ "`command -v bundle`" == "" ]; then + echo "WARN: Could not find bundle." + echo "Attempting to install locally. If this doesn't work, please install with 'gem install bundler'." + + # install bundler locally + ${GEM} install --user-install --no-format-executable bundler + fi fi # Install Ruby dependencies locally @@ -57,7 +59,6 @@ while getopts "pi" opt; do ;; i) [[ `${RUBY} -v` =~ 'ruby 1' ]] && echo "Error: building the docs with the incremental option requires at least ruby 2.0" && exit 1 - bundle install --path .rubydeps JEKYLL_CMD="liveserve --baseurl= --watch --incremental" ;; esac diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index cb6d58f0c1e97b..1c7c2816a0e145 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -31,7 +31,7 @@ under the License. For distributed execution, Flink *chains* operator subtasks together into *tasks*. Each task is executed by one thread. Chaining operators together into tasks is a useful optimization: it reduces the overhead of thread-to-thread handover and buffering, and increases overall throughput while decreasing latency. -The chaining behavior can be configured; see the [chaining docs](../dev/datastream_api.html#task-chaining-and-resource-groups) for details. +The chaining behavior can be configured; see the [chaining docs](../dev/stream/operators/#task-chaining-and-resource-groups) for details. The sample dataflow in the figure below is executed with five subtasks, and hence with five parallel threads. @@ -98,7 +98,7 @@ job. Allowing this *slot sharing* has two main benefits: TaskManagers with shared Task Slots -The APIs also include a *[resource group](../dev/datastream_api.html#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing. +The APIs also include a *[resource group](../dev/stream/operators/#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing. As a rule-of-thumb, a good default number of task slots would be the number of CPU cores. With hyper-threading, each slot then takes 2 or more hardware thread contexts. diff --git a/docs/dev/api_concepts.md b/docs/dev/api_concepts.md index c4215074683259..bd7ca5aff90271 100644 --- a/docs/dev/api_concepts.md +++ b/docs/dev/api_concepts.md @@ -510,7 +510,7 @@ data.map(new MapFunction () { #### Java 8 Lambdas -Flink also supports Java 8 Lambdas in the Java API. Please see the full [Java 8 Guide]({{ site.baseurl }}/dev/java8.html). +Flink also supports Java 8 Lambdas in the Java API. {% highlight java %} data.filter(s -> s.startsWith("http://")); diff --git a/docs/dev/batch/connectors.md b/docs/dev/batch/connectors.md index 23917448f39c78..7d13d1e2a34ab2 100644 --- a/docs/dev/batch/connectors.md +++ b/docs/dev/batch/connectors.md @@ -27,7 +27,7 @@ under the License. ## Reading from file systems -Flink has build-in support for the following file systems: +Flink has built-in support for the following file systems: | Filesystem | Scheme | Notes | | ------------------------------------- |--------------| ------ | @@ -64,12 +64,12 @@ See [Deployment & Operations - Deployment - AWS - S3: Simple Storage Service]({{ For Alluxio support add the following entry into the `core-site.xml` file: -~~~xml +{% highlight xml %} fs.alluxio.impl alluxio.hadoop.FileSystem -~~~ +{% endhighlight %} ## Connecting to other systems using Input/OutputFormat wrappers for Hadoop @@ -86,31 +86,31 @@ This section shows some examples for connecting Flink to other systems. ## Avro support in Flink -Flink has extensive build-in support for [Apache Avro](http://avro.apache.org/). This allows to easily read from Avro files with Flink. +Flink has extensive built-in support for [Apache Avro](http://avro.apache.org/). This allows to easily read from Avro files with Flink. Also, the serialization framework of Flink is able to handle classes generated from Avro schemas. Be sure to include the Flink Avro dependency to the pom.xml of your project. -~~~xml +{% highlight xml %} org.apache.flink flink-avro {{site.version }} -~~~ +{% endhighlight %} In order to read data from an Avro file, you have to specify an `AvroInputFormat`. **Example**: -~~~java +{% highlight java %} AvroInputFormat users = new AvroInputFormat(in, User.class); DataSet usersDS = env.createInput(users); -~~~ +{% endhighlight %} Note that `User` is a POJO generated by Avro. Flink also allows to perform string-based key selection of these POJOs. For example: -~~~java +{% highlight java %} usersDS.groupBy("name") -~~~ +{% endhighlight %} Note that using the `GenericData.Record` type is possible with Flink, but not recommended. Since the record contains the full schema, its very data intensive and thus probably slow to use. @@ -129,21 +129,21 @@ This example is using the `HadoopInputFormat` wrapper to use an existing Hadoop 1. Download and compile the `azure-tables-hadoop` project. The input format developed by the project is not yet available in Maven Central, therefore, we have to build the project ourselves. Execute the following commands: - ~~~bash + {% highlight bash %} git clone https://github.com/mooso/azure-tables-hadoop.git cd azure-tables-hadoop mvn clean install - ~~~ + {% endhighlight %} 2. Setup a new Flink project using the quickstarts: - ~~~bash + {% highlight bash %} curl https://flink.apache.org/q/quickstart.sh | bash - ~~~ + {% endhighlight %} 3. Add the following dependencies (in the `` section) to your `pom.xml` file: - ~~~xml + {% highlight xml %} org.apache.flink flink-hadoop-compatibility{{ site.scala_version_suffix }} @@ -154,7 +154,7 @@ Execute the following commands: microsoft-hadoop-azure 0.0.4 - ~~~ + {% endhighlight %} `flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers. `microsoft-hadoop-azure` is adding the project we've build before to our project. @@ -164,7 +164,7 @@ Browse to the code of the `Job.java` file. Its an empty skeleton for a Flink job Paste the following code into it: -~~~java +{% highlight java %} import java.util.Map; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.DataSet; @@ -217,7 +217,7 @@ public class AzureTableExample { env.execute("Azure Example"); } } -~~~ +{% endhighlight %} The example shows how to access an Azure table and turn data into Flink's `DataSet` (more specifically, the type of the set is `DataSet>`). With the `DataSet`, you can apply all known transformations to the DataSet. diff --git a/docs/dev/batch/dataset_transformations.md b/docs/dev/batch/dataset_transformations.md index d63ee88ccd1f95..cf2da3fa91a0cd 100644 --- a/docs/dev/batch/dataset_transformations.md +++ b/docs/dev/batch/dataset_transformations.md @@ -42,7 +42,7 @@ The following code transforms a DataSet of Integer pairs into a DataSet of Integ
    -~~~java +{% highlight java %} // MapFunction that adds two integer values public class IntAdder implements MapFunction, Integer> { @Override @@ -54,22 +54,22 @@ public class IntAdder implements MapFunction, Integer> // [...] DataSet> intPairs = // [...] DataSet intSums = intPairs.map(new IntAdder()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val intPairs: DataSet[(Int, Int)] = // [...] val intSums = intPairs.map { pair => pair._1 + pair._2 } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} intSums = intPairs.map(lambda x: sum(x)) -~~~ +{% endhighlight %}
    @@ -84,7 +84,7 @@ The following code transforms a DataSet of text lines into a DataSet of words:
    -~~~java +{% highlight java %} // FlatMapFunction that tokenizes a String by whitespace characters and emits all String tokens. public class Tokenizer implements FlatMapFunction { @Override @@ -98,22 +98,22 @@ public class Tokenizer implements FlatMapFunction { // [...] DataSet textLines = // [...] DataSet words = textLines.flatMap(new Tokenizer()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val textLines: DataSet[String] = // [...] val words = textLines.flatMap { _.split(" ") } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} words = lines.flat_map(lambda x,c: [line.split() for line in x]) -~~~ +{% endhighlight %}
    @@ -129,7 +129,7 @@ The following code transforms a DataSet of text lines into a DataSet of counts p
    -~~~java +{% highlight java %} public class PartitionCounter implements MapPartitionFunction { public void mapPartition(Iterable values, Collector out) { @@ -144,24 +144,24 @@ public class PartitionCounter implements MapPartitionFunction { // [...] DataSet textLines = // [...] DataSet counts = textLines.mapPartition(new PartitionCounter()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val textLines: DataSet[String] = // [...] // Some is required because the return value must be a Collection. // There is an implicit conversion from Option to a Collection. val counts = texLines.mapPartition { in => Some(in.size) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} counts = lines.map_partition(lambda x,c: [sum(1 for _ in x)]) -~~~ +{% endhighlight %}
    @@ -175,7 +175,7 @@ The following code removes all Integers smaller than zero from a DataSet:
    -~~~java +{% highlight java %} // FilterFunction that filters out all Integers smaller than zero. public class NaturalNumberFilter implements FilterFunction { @Override @@ -187,22 +187,22 @@ public class NaturalNumberFilter implements FilterFunction { // [...] DataSet intNumbers = // [...] DataSet naturalNumbers = intNumbers.filter(new NaturalNumberFilter()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val intNumbers: DataSet[Int] = // [...] val naturalNumbers = intNumbers.filter { _ > 0 } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} naturalNumbers = intNumbers.filter(lambda x: x > 0) -~~~ +{% endhighlight %}
    @@ -222,40 +222,40 @@ The following code shows different ways to apply a Project transformation on a D
    -~~~java +{% highlight java %} DataSet> in = // [...] // converts Tuple3 into Tuple2 DataSet> out = in.project(2,0); -~~~ +{% endhighlight %} #### Projection with Type Hint Note that the Java compiler cannot infer the return type of `project` operator. This can cause a problem if you call another operator on a result of `project` operator such as: -~~~java +{% highlight java %} DataSet> ds = .... DataSet> ds2 = ds.project(0).distinct(0); -~~~ +{% endhighlight %} This problem can be overcome by hinting the return type of `project` operator like this: -~~~java +{% highlight java %} DataSet> ds2 = ds.>project(0).distinct(0); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} Not supported. -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} out = in.project(2,0); -~~~ +{% endhighlight %}
    @@ -294,7 +294,7 @@ with a reduce function.
    -~~~java +{% highlight java %} // some ordinary POJO public class WC { public String word; @@ -317,12 +317,12 @@ DataSet wordCounts = words .groupBy("word") // apply ReduceFunction on grouped DataSet .reduce(new WordCounter()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // some ordinary POJO class WC(val word: String, val count: Int) { def this() { @@ -335,14 +335,14 @@ val words: DataSet[WC] = // [...] val wordCounts = words.groupBy("word").reduce { (w1, w2) => new WC(w1.word, w1.count + w2.count) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -356,7 +356,7 @@ with a reduce function.
    -~~~java +{% highlight java %} // some ordinary POJO public class WC { public String word; @@ -386,12 +386,12 @@ public class SelectWord implements KeySelector { return w.word; } } -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // some ordinary POJO class WC(val word: String, val count: Int) { def this() { @@ -404,12 +404,12 @@ val words: DataSet[WC] = // [...] val wordCounts = words.groupBy { _.word } reduce { (w1, w2) => new WC(w1.word, w1.count + w2.count) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class WordCounter(ReduceFunction): def reduce(self, in1, in2): return (in1[0], in1[1] + in2[1]) @@ -418,7 +418,7 @@ words = // [...] wordCounts = words \ .group_by(lambda x: x[0]) \ .reduce(WordCounter()) -~~~ +{% endhighlight %}
    @@ -430,30 +430,30 @@ The following code shows how to use field position keys and apply a reduce funct
    -~~~java +{% highlight java %} DataSet> tuples = // [...] DataSet> reducedTuples = tuples // group DataSet on first and second field of Tuple .groupBy(0, 1) // apply ReduceFunction on grouped DataSet .reduce(new MyTupleReducer()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val tuples = DataSet[(String, Int, Double)] = // [...] // group on the first and second Tuple field val reducedTuples = tuples.groupBy(0, 1).reduce { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} reducedTuples = tuples.group_by(0, 1).reduce( ... ) -~~~ +{% endhighlight %}
    @@ -465,25 +465,25 @@ When using Case Classes you can also specify the grouping key using the names of
    -~~~java +{% highlight java %} Not supported. -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} case class MyClass(val a: String, b: Int, c: Double) val tuples = DataSet[MyClass] = // [...] // group on the first and second field val reducedTuples = tuples.groupBy("a", "b").reduce { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -502,7 +502,7 @@ The following code shows how duplicate strings can be removed from a DataSet gro
    -~~~java +{% highlight java %} public class DistinctReduce implements GroupReduceFunction, Tuple2> { @@ -530,23 +530,23 @@ DataSet> input = // [...] DataSet> output = input .groupBy(0) // group DataSet by the first tuple field .reduceGroup(new DistinctReduce()); // apply GroupReduceFunction -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String)] = // [...] val output = input.groupBy(0).reduceGroup { (in, out: Collector[(Int, String)]) => in.toSet foreach (out.collect) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class DistinctReduce(GroupReduceFunction): def reduce(self, iterator, collector): dic = dict() @@ -556,7 +556,7 @@ val output = input.groupBy(0).reduceGroup { collector.collect(key) output = data.group_by(0).reduce_group(DistinctReduce()) -~~~ +{% endhighlight %}
    @@ -578,7 +578,7 @@ The following code shows another example how to remove duplicate Strings in a Da
    -~~~java +{% highlight java %} // GroupReduceFunction that removes consecutive identical elements public class DistinctReduce implements GroupReduceFunction, Tuple2> { @@ -607,12 +607,12 @@ DataSet output = input .groupBy(0) // group DataSet by first field .sortGroup(1, Order.ASCENDING) // sort groups on second tuple field .reduceGroup(new DistinctReduce()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String)] = // [...] val output = input.groupBy(0).sortGroup(1, Order.ASCENDING).reduceGroup { (in, out: Collector[(Int, String)]) => @@ -624,12 +624,12 @@ val output = input.groupBy(0).sortGroup(1, Order.ASCENDING).reduceGroup { } } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class DistinctReduce(GroupReduceFunction): def reduce(self, iterator, collector): dic = dict() @@ -639,7 +639,7 @@ val output = input.groupBy(0).sortGroup(1, Order.ASCENDING).reduceGroup { collector.collect(key) output = data.group_by(0).sort_group(1, Order.ASCENDING).reduce_group(DistinctReduce()) -~~~ +{% endhighlight %}
    @@ -660,7 +660,7 @@ of the `GroupReduceFunction` as shown in the following example:
    -~~~java +{% highlight java %} // Combinable GroupReduceFunction that computes a sum. public class MyCombinableGroupReducer implements GroupReduceFunction, String>, @@ -695,12 +695,12 @@ public class MyCombinableGroupReducer implements out.collect(new Tuple2<>(key, sum)); } } -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // Combinable GroupReduceFunction that computes two sums. class MyCombinableGroupReducer @@ -712,7 +712,7 @@ class MyCombinableGroupReducer out: Collector[String]): Unit = { val r: (String, Int) = - in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) + in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) // concat key and sum and emit out.collect (r._1 + "-" + r._2) } @@ -722,17 +722,17 @@ class MyCombinableGroupReducer out: Collector[(String, Int)]): Unit = { val r: (String, Int) = - in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) + in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) ) // emit tuple with key and sum out.collect(r) } } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class GroupReduce(GroupReduceFunction): def reduce(self, iterator, collector): key, int_sum = iterator.next() @@ -747,7 +747,7 @@ class MyCombinableGroupReducer collector.collect((key, int_sum)) data.reduce_group(GroupReduce(), combinable=True) -~~~ +{% endhighlight %}
    @@ -778,7 +778,7 @@ an alternative WordCount implementation.
    -~~~java +{% highlight java %} DataSet input = [..] // The words received as input DataSet> combinedWords = input @@ -814,12 +814,12 @@ DataSet> output = combinedWords out.collect(new Tuple2(key, count)); } }); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[String] = [..] // The words received as input val combinedWords: DataSet[(String, Int)] = input @@ -850,14 +850,14 @@ val output: DataSet[(String, Int)] = combinedWords out.collect((key, sum)) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -883,31 +883,31 @@ The following code shows how to apply an Aggregation transformation on a DataSet
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input .groupBy(1) // group DataSet on second field .aggregate(SUM, 0) // compute sum of the first field .and(MIN, 2); // compute minimum of the third field -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String, Double)] = // [...] val output = input.groupBy(1).aggregate(SUM, 0).and(MIN, 2) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} from flink.functions.Aggregation import Sum, Min input = # [...] output = input.group_by(1).aggregate(Sum, 0).and_agg(Min, 2) -~~~ +{% endhighlight %}
    @@ -926,29 +926,29 @@ The following code shows how to select the tuple with the minimum values for the
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input .groupBy(1) // group DataSet on second field .minBy(0, 2); // select tuple with minimum values for first and third field. -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String, Double)] = // [...] val output: DataSet[(Int, String, Double)] = input .groupBy(1) // group DataSet on second field .minBy(0, 2) // select tuple with minimum values for first and third field. -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -963,7 +963,7 @@ The following code shows how to sum all elements of an Integer DataSet:
    -~~~java +{% highlight java %} // ReduceFunction that sums Integers public class IntSummer implements ReduceFunction { @Override @@ -975,23 +975,23 @@ public class IntSummer implements ReduceFunction { // [...] DataSet intNumbers = // [...] DataSet sum = intNumbers.reduce(new IntSummer()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val intNumbers = env.fromElements(1,2,3) val sum = intNumbers.reduce (_ + _) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} intNumbers = env.from_elements(1,2,3) sum = intNumbers.reduce(lambda x,y: x + y) -~~~ +{% endhighlight %}
    @@ -1008,26 +1008,26 @@ The following example shows how to apply a GroupReduce transformation on a full
    -~~~java +{% highlight java %} DataSet input = // [...] // apply a (preferably combinable) GroupReduceFunction to a DataSet DataSet output = input.reduceGroup(new MyGroupReducer()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[Int] = // [...] val output = input.reduceGroup(new MyGroupReducer()) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} output = data.reduce_group(MyGroupReducer()) -~~~ +{% endhighlight %}
    @@ -1059,31 +1059,31 @@ The following code shows how to apply an Aggregation transformation on a full Da
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input .aggregate(SUM, 0) // compute sum of the first field .and(MIN, 1); // compute minimum of the second field -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String, Double)] = // [...] val output = input.aggregate(SUM, 0).and(MIN, 2) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} from flink.functions.Aggregation import Sum, Min input = # [...] output = input.aggregate(Sum, 0).and_agg(Min, 2) -~~~ +{% endhighlight %}
    @@ -1099,27 +1099,27 @@ The following code shows how to select the tuple with the maximum values for the
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input .maxBy(0, 2); // select tuple with maximum values for first and third field. -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String, Double)] = // [...] val output: DataSet[(Int, String, Double)] = input .maxBy(0, 2) // select tuple with maximum values for first and third field. -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1132,27 +1132,27 @@ The following code removes all duplicate elements from the DataSet:
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input.distinct(); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, String, Double)] = // [...] val output = input.distinct() -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1168,27 +1168,27 @@ It is also possible to change how the distinction of the elements in the DataSet
    -~~~java +{% highlight java %} DataSet> input = // [...] DataSet> output = input.distinct(0,2); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[(Int, Double, String)] = // [...] val output = input.distinct(0,2) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1198,7 +1198,7 @@ Not supported.
    -~~~java +{% highlight java %} private static class AbsSelector implements KeySelector { private static final long serialVersionUID = 1L; @Override @@ -1209,23 +1209,23 @@ private static final long serialVersionUID = 1L; DataSet input = // [...] DataSet output = input.distinct(new AbsSelector()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input: DataSet[Int] = // [...] val output = input.distinct {x => Math.abs(x)} -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1235,7 +1235,7 @@ Not supported.
    -~~~java +{% highlight java %} // some ordinary POJO public class CustomType { public String aName; @@ -1246,26 +1246,26 @@ public class CustomType { DataSet input = // [...] DataSet output = input.distinct("aName", "aNumber"); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // some ordinary POJO case class CustomType(aName : String, aNumber : Int) { } val input: DataSet[CustomType] = // [...] val output = input.distinct("aName", "aNumber") -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1275,28 +1275,28 @@ It is also possible to indicate to use all the fields by the wildcard character:
    -~~~java +{% highlight java %} DataSet input = // [...] DataSet output = input.distinct("*"); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // some ordinary POJO val input: DataSet[CustomType] = // [...] val output = input.distinct("_") -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1321,7 +1321,7 @@ The following code shows a default Join transformation using field position keys
    -~~~java +{% highlight java %} public static class User { public String name; public int zip; } public static class Store { public Manager mgr; public int zip; } DataSet input1 = // [...] @@ -1331,23 +1331,23 @@ DataSet> result = input1.join(input2) .where("zip") // key of the first input (users) .equalTo("zip"); // key of the second input (stores) -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input1: DataSet[(Int, String)] = // [...] val input2: DataSet[(Double, Int)] = // [...] val result = input1.join(input2).where(0).equalTo(1) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} result = input1.join(input2).where(0).equal_to(1) -~~~ +{% endhighlight %}
    @@ -1362,7 +1362,7 @@ The following code performs a join of DataSet with custom java objects and a Tup
    -~~~java +{% highlight java %} // some POJO public class Rating { public String name; @@ -1395,12 +1395,12 @@ DataSet> // applying the JoinFunction on joining pairs .with(new PointWeighter()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} case class Rating(name: String, category: String, points: Int) val ratings: DataSet[Ratings] = // [...] @@ -1409,12 +1409,12 @@ val weights: DataSet[(String, Double)] = // [...] val weightedRatings = ratings.join(weights).where("category").equalTo(0) { (rating, weight) => (rating.name, rating.points * weight._2) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class PointWeighter(JoinFunction): def join(self, rating, weight): return (rating[0], rating[1] * weight[1]) @@ -1423,7 +1423,7 @@ val weightedRatings = ratings.join(weights).where("category").equalTo(0) { weightedRatings = ratings.join(weights).where(0).equal_to(0). \ with(new PointWeighter()); -~~~ +{% endhighlight %}
    @@ -1437,7 +1437,7 @@ return (collect), zero, one, or more elements.
    -~~~java +{% highlight java %} public class PointWeighter implements FlatJoinFunction, Tuple2> { @Override @@ -1452,12 +1452,12 @@ public class PointWeighter DataSet> weightedRatings = ratings.join(weights) // [...] -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} case class Rating(name: String, category: String, points: Int) val ratings: DataSet[Ratings] = // [...] @@ -1468,7 +1468,7 @@ val weightedRatings = ratings.join(weights).where("category").equalTo(0) { if (weight._2 > 0.1) out.collect(rating.name, rating.points * weight._2) } -~~~ +{% endhighlight %}
    @@ -1483,10 +1483,10 @@ A Join transformation can construct result tuples using a projection as shown he
    -~~~java +{% highlight java %} DataSet> input1 = // [...] DataSet> input2 = // [...] -DataSet +DataSet> result = input1.join(input2) // key definition on first DataSet using a field position key @@ -1495,7 +1495,7 @@ DataSet .equalTo(0) // select and reorder fields of matching tuples .projectFirst(0,2).projectSecond(1).projectFirst(1); -~~~ +{% endhighlight %} `projectFirst(int...)` and `projectSecond(int...)` select the fields of the first and second joined input that should be assembled into an output Tuple. The order of indexes defines the order of fields in the output tuple. The join projection works also for non-Tuple DataSets. In this case, `projectFirst()` or `projectSecond()` must be called without arguments to add a joined element to the output Tuple. @@ -1503,17 +1503,17 @@ The join projection works also for non-Tuple DataSets. In this case, `projectFir
    -~~~scala +{% highlight scala %} Not supported. -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} result = input1.join(input2).where(0).equal_to(0) \ .project_first(0,2).project_second(1).project_first(1); -~~~ +{% endhighlight %} `project_first(int...)` and `project_second(int...)` select the fields of the first and second joined input that should be assembled into an output Tuple. The order of indexes defines the order of fields in the output tuple. The join projection works also for non-Tuple DataSets. In this case, `project_first()` or `project_second()` must be called without arguments to add a joined element to the output Tuple. @@ -1528,7 +1528,7 @@ In order to guide the optimizer to pick the right execution strategy, you can hi
    -~~~java +{% highlight java %} DataSet> input1 = // [...] DataSet> input2 = // [...] @@ -1545,12 +1545,12 @@ DataSet, Tuple2>> input1.joinWithHuge(input2) .where(0) .equalTo(0); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input1: DataSet[(Int, String)] = // [...] val input2: DataSet[(Int, String)] = // [...] @@ -1560,12 +1560,12 @@ val result1 = input1.joinWithTiny(input2).where(0).equalTo(0) // hint that the second DataSet is very large val result1 = input1.joinWithHuge(input2).where(0).equalTo(0) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} #hint that the second DataSet is very small result1 = input1.join_with_tiny(input2).where(0).equal_to(0) @@ -1573,7 +1573,7 @@ val result1 = input1.joinWithHuge(input2).where(0).equalTo(0) #hint that the second DataSet is very large result1 = input1.join_with_huge(input2).where(0).equal_to(0) -~~~ +{% endhighlight %}
    @@ -1587,33 +1587,33 @@ to manually pick a strategy, in case you want to enforce a specific way of execu
    -~~~java +{% highlight java %} DataSet input1 = // [...] DataSet input2 = // [...] DataSet result = input1.join(input2, JoinHint.BROADCAST_HASH_FIRST) .where("id").equalTo("key"); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input1: DataSet[SomeType] = // [...] val input2: DataSet[AnotherType] = // [...] // hint that the second DataSet is very small val result1 = input1.join(input2, JoinHint.BROADCAST_HASH_FIRST).where("id").equalTo("key") -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1668,7 +1668,7 @@ The following code performs a left outer join of DataSet with custom java object
    -~~~java +{% highlight java %} // some POJO public class Rating { public String name; @@ -1702,12 +1702,12 @@ DataSet> // applying the JoinFunction on joining pairs .with(new PointAssigner()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} case class Rating(name: String, category: String, points: Int) val movies: DataSet[(String, String)] = // [...] @@ -1716,14 +1716,14 @@ val ratings: DataSet[Ratings] = // [...] val moviesWithPoints = movies.leftOuterJoin(ratings).where(0).equalTo("name") { (movie, rating) => (movie._1, if (rating == null) -1 else rating.points) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1737,7 +1737,7 @@ return (collect), zero, one, or more elements.
    -~~~java +{% highlight java %} public class PointAssigner implements FlatJoinFunction, Rating, Tuple2> { @Override @@ -1755,21 +1755,21 @@ public class PointAssigner DataSet> moviesWithPoints = movies.leftOuterJoin(ratings) // [...] -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} Not supported. -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1783,7 +1783,7 @@ to manually pick a strategy, in case you want to enforce a specific way of execu
    -~~~java +{% highlight java %} DataSet input1 = // [...] DataSet input2 = // [...] @@ -1794,12 +1794,12 @@ DataSet result1 = DataSet result2 = input1.rightOuterJoin(input2, JoinHint.BROADCAST_HASH_FIRST) .where("id").equalTo("key"); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input1: DataSet[SomeType] = // [...] val input2: DataSet[AnotherType] = // [...] @@ -1808,14 +1808,14 @@ val result1 = input1.leftOuterJoin(input2, JoinHint.REPARTITION_SORT_MERGE).wher val result2 = input1.rightOuterJoin(input2, JoinHint.BROADCAST_HASH_FIRST).where("id").equalTo("key") -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -1878,7 +1878,7 @@ The following code shows how to apply a Cross transformation on two DataSets usi
    -~~~java +{% highlight java %} public class Coord { public int id; public int x; @@ -1904,13 +1904,13 @@ DataSet> coords1.cross(coords2) // apply CrossFunction .with(new EuclideanDistComputer()); -~~~ +{% endhighlight %} #### Cross with Projection A Cross transformation can also construct result tuples using a projection as shown here: -~~~java +{% highlight java %} DataSet> input1 = // [...] DataSet> input2 = // [...] DataSet @@ -1918,14 +1918,14 @@ DataSet input1.cross(input2) // select and reorder fields of matching tuples .projectSecond(0).projectFirst(1,0).projectSecond(1); -~~~ +{% endhighlight %} The field selection in a Cross projection works the same way as in the projection of Join results.
    -~~~scala +{% highlight scala %} case class Coord(id: Int, x: Int, y: Int) val coords1: DataSet[Coord] = // [...] @@ -1936,27 +1936,27 @@ val distances = coords1.cross(coords2) { val dist = sqrt(pow(c1.x - c2.x, 2) + pow(c1.y - c2.y, 2)) (c1.id, c2.id, dist) } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class Euclid(CrossFunction): def cross(self, c1, c2): return (c1[0], c2[0], sqrt(pow(c1[1] - c2.[1], 2) + pow(c1[2] - c2[2], 2))) distances = coords1.cross(coords2).using(Euclid()) -~~~ +{% endhighlight %} #### Cross with Projection A Cross transformation can also construct result tuples using a projection as shown here: -~~~python +{% highlight python %} result = input1.cross(input2).projectFirst(1,0).projectSecond(0,1); -~~~ +{% endhighlight %} The field selection in a Cross projection works the same way as in the projection of Join results. @@ -1970,7 +1970,7 @@ In order to guide the optimizer to pick the right execution strategy, you can hi
    -~~~java +{% highlight java %} DataSet> input1 = // [...] DataSet> input2 = // [...] @@ -1987,12 +1987,12 @@ DataSet> input1.crossWithHuge(input2) // apply a projection (or any Cross function) .projectFirst(0,1).projectSecond(1); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val input1: DataSet[(Int, String)] = // [...] val input2: DataSet[(Int, String)] = // [...] @@ -2002,19 +2002,19 @@ val result1 = input1.crossWithTiny(input2) // hint that the second DataSet is very large val result1 = input1.crossWithHuge(input2) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} #hint that the second DataSet is very small result1 = input1.cross_with_tiny(input2) #hint that the second DataSet is very large result1 = input1.cross_with_huge(input2) -~~~ +{% endhighlight %}
    @@ -2033,7 +2033,7 @@ Similar to Reduce, GroupReduce, and Join, keys can be defined using the differen The example shows how to group by Field Position Keys (Tuple DataSets only). You can do the same with Pojo-types and key expressions. -~~~java +{% highlight java %} // Some CoGroupFunction definition class MyCoGrouper implements CoGroupFunction, Tuple2, Double> { @@ -2069,12 +2069,12 @@ DataSet output = iVals.coGroup(dVals) .equalTo(0) // apply CoGroup function on each pair of groups .with(new MyCoGrouper()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val iVals: DataSet[(String, Int)] = // [...] val dVals: DataSet[(String, Double)] = // [...] @@ -2088,12 +2088,12 @@ val output = iVals.coGroup(dVals).where(0).equalTo(0) { } } } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} class CoGroup(CoGroupFunction): def co_group(self, ivals, dvals, collector): ints = dict() @@ -2107,7 +2107,7 @@ val output = iVals.coGroup(dVals).where(0).equalTo(0) { output = ivals.co_group(dvals).where(0).equal_to(0).using(CoGroup()) -~~~ +{% endhighlight %}
    @@ -2120,30 +2120,30 @@ Produces the union of two DataSets, which have to be of the same type. A union o
    -~~~java +{% highlight java %} DataSet> vals1 = // [...] DataSet> vals2 = // [...] DataSet> vals3 = // [...] DataSet> unioned = vals1.union(vals2).union(vals3); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val vals1: DataSet[(String, Int)] = // [...] val vals2: DataSet[(String, Int)] = // [...] val vals3: DataSet[(String, Int)] = // [...] val unioned = vals1.union(vals2).union(vals3) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} unioned = vals1.union(vals2).union(vals3) -~~~ +{% endhighlight %}
    @@ -2154,28 +2154,28 @@ Evenly rebalances the parallel partitions of a DataSet to eliminate data skew.
    -~~~java +{% highlight java %} DataSet in = // [...] // rebalance DataSet and apply a Map transformation. DataSet> out = in.rebalance() .map(new Mapper()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val in: DataSet[String] = // [...] // rebalance DataSet and apply a Map transformation. val out = in.rebalance().map { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -2189,28 +2189,28 @@ Keys can be specified as position keys, expression keys, and key selector functi
    -~~~java +{% highlight java %} DataSet> in = // [...] // hash-partition DataSet by String value and apply a MapPartition transformation. DataSet> out = in.partitionByHash(0) .mapPartition(new PartitionMapper()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val in: DataSet[(String, Int)] = // [...] // hash-partition DataSet by String value and apply a MapPartition transformation. val out = in.partitionByHash(0).mapPartition { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -2223,28 +2223,28 @@ Keys can be specified as position keys, expression keys, and key selector functi
    -~~~java +{% highlight java %} DataSet> in = // [...] // range-partition DataSet by String value and apply a MapPartition transformation. DataSet> out = in.partitionByRange(0) .mapPartition(new PartitionMapper()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val in: DataSet[(String, Int)] = // [...] // range-partition DataSet by String value and apply a MapPartition transformation. val out = in.partitionByRange(0).mapPartition { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -2259,7 +2259,7 @@ Partitions can be sorted on multiple fields by chaining `sortPartition()` calls.
    -~~~java +{% highlight java %} DataSet> in = // [...] // Locally sort partitions in ascending order on the second String field and // in descending order on the first String field. @@ -2267,12 +2267,12 @@ DataSet> in = // [...] DataSet> out = in.sortPartition(1, Order.ASCENDING) .sortPartition(0, Order.DESCENDING) .mapPartition(new PartitionMapper()); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val in: DataSet[(String, Int)] = // [...] // Locally sort partitions in ascending order on the second String field and // in descending order on the first String field. @@ -2280,14 +2280,14 @@ val in: DataSet[(String, Int)] = // [...] val out = in.sortPartition(1, Order.ASCENDING) .sortPartition(0, Order.DESCENDING) .mapPartition { ... } -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    @@ -2299,7 +2299,7 @@ Returns the first n (arbitrary) elements of a DataSet. First-n can be applied on
    -~~~java +{% highlight java %} DataSet> in = // [...] // Return the first five (arbitrary) elements of the DataSet DataSet> out1 = in.first(5); @@ -2312,12 +2312,12 @@ DataSet> out2 = in.groupBy(0) DataSet> out3 = in.groupBy(0) .sortGroup(1, Order.ASCENDING) .first(3); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val in: DataSet[(String, Int)] = // [...] // Return the first five (arbitrary) elements of the DataSet val out1 = in.first(5) @@ -2327,14 +2327,14 @@ val out2 = in.groupBy(0).first(2) // Return the first three elements of each String group ordered by the Integer field val out3 = in.groupBy(0).sortGroup(1, Order.ASCENDING).first(3) -~~~ +{% endhighlight %}
    -~~~python +{% highlight python %} Not supported. -~~~ +{% endhighlight %}
    diff --git a/docs/dev/batch/examples.md b/docs/dev/batch/examples.md index a4b282688eed4a..fe2bd8d3bdc78d 100644 --- a/docs/dev/batch/examples.md +++ b/docs/dev/batch/examples.md @@ -27,8 +27,7 @@ The following example programs showcase different applications of Flink from simple word counting to graph algorithms. The code samples illustrate the use of [Flink's DataSet API]({{ site.baseurl }}/dev/batch/index.html). -The full source code of the following and more examples can be found in the __flink-examples-batch__ -or __flink-examples-streaming__ module of the Flink source repository. +The full source code of the following and more examples can be found in the {% gh_link flink-examples/flink-examples-batch "flink-examples-batch" %} module of the Flink source repository. * This will be replaced by the TOC {:toc} @@ -44,17 +43,17 @@ Each binary release of Flink contains an `examples` directory with jar files for To run the WordCount example, issue the following command: -~~~bash +{% highlight bash %} ./bin/flink run ./examples/batch/WordCount.jar -~~~ +{% endhighlight %} The other examples can be started in a similar way. Note that many examples run without passing any arguments for them, by using build-in data. To run WordCount with real data, you have to pass the path to the data: -~~~bash +{% highlight bash %} ./bin/flink run ./examples/batch/WordCount.jar --input /path/to/some/text/data --output /path/to/result -~~~ +{% endhighlight %} Note that non-local file systems require a schema prefix, such as `hdfs://`. @@ -65,7 +64,7 @@ WordCount is the "Hello World" of Big Data processing systems. It computes the f
    -~~~java +{% highlight java %} ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet text = env.readTextFile("/path/to/file"); @@ -95,14 +94,14 @@ public static class Tokenizer implements FlatMapFunction --output `. As test data, any text file will do.
    -~~~scala +{% highlight scala %} val env = ExecutionEnvironment.getExecutionEnvironment // get input data @@ -114,7 +113,7 @@ val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } } .sum(1) counts.writeAsCsv(outputPath, "\n", " ") -~~~ +{% endhighlight %} The {% gh_link /flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala "WordCount example" %} implements the above described algorithm with input parameters: `--input --output `. As test data, any text file will do. @@ -131,7 +130,7 @@ In this simple example, PageRank is implemented with a [bulk iteration](iteratio
    -~~~java +{% highlight java %} ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // read the pages and initial ranks by parsing a CSV file @@ -202,7 +201,7 @@ public static final class EpsilonFilter return Math.abs(value.f0.f1 - value.f1.f1) > EPSILON; } } -~~~ +{% endhighlight %} The {% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/graph/PageRank.java "PageRank program" %} implements the above example. It requires the following parameters to run: `--pages --links --output --numPages --iterations `. @@ -210,7 +209,7 @@ It requires the following parameters to run: `--pages --links --ou
    -~~~scala +{% highlight scala %} // User-defined types case class Link(sourceId: Long, targetId: Long) case class Page(pageId: Long, rank: Double) @@ -269,7 +268,7 @@ val result = finalRanks // emit result result.writeAsCsv(outputPath, "\n", " ") -~~~ +{% endhighlight %} he {% gh_link /flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala "PageRank program" %} implements the above example. It requires the following parameters to run: `--pages --links --output --numPages --iterations `. @@ -293,7 +292,7 @@ This implementation uses a [delta iteration](iterations.html): Vertices that hav
    -~~~java +{% highlight java %} // read vertex and edge data DataSet vertices = getVertexDataSet(env); DataSet> edges = getEdgeDataSet(env).flatMap(new UndirectEdge()); @@ -365,14 +364,14 @@ public static final class ComponentIdFilter } } } -~~~ +{% endhighlight %} The {% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/graph/ConnectedComponents.java "ConnectedComponents program" %} implements the above example. It requires the following parameters to run: `--vertices --edges --output --iterations `.
    -~~~scala +{% highlight scala %} // set up execution environment val env = ExecutionEnvironment.getExecutionEnvironment @@ -408,7 +407,7 @@ val verticesWithComponents = vertices.iterateDelta(vertices, maxIterations, Arra verticesWithComponents.writeAsCsv(outputPath, "\n", " ") -~~~ +{% endhighlight %} The {% gh_link /flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala "ConnectedComponents program" %} implements the above example. It requires the following parameters to run: `--vertices --edges --output --iterations `.
    @@ -420,102 +419,4 @@ Input files are plain text files and must be formatted as follows: - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges are separated by new-line characters: * For example `"1 2\n2 12\n1 12\n42 63\n"` gives four (undirected) links (1)-(2), (2)-(12), (1)-(12), and (42)-(63). -## Relational Query - -The Relational Query example assumes two tables, one with `orders` and the other with `lineitems` as specified by the [TPC-H decision support benchmark](http://www.tpc.org/tpch/). TPC-H is a standard benchmark in the database industry. See below for instructions how to generate the input data. - -The example implements the following SQL query. - -~~~sql -SELECT l_orderkey, o_shippriority, sum(l_extendedprice) as revenue - FROM orders, lineitem -WHERE l_orderkey = o_orderkey - AND o_orderstatus = "F" - AND YEAR(o_orderdate) > 1993 - AND o_orderpriority LIKE "5%" -GROUP BY l_orderkey, o_shippriority; -~~~ - -The Flink program, which implements the above query looks as follows. - -
    -
    - -~~~java -// get orders data set: (orderkey, orderstatus, orderdate, orderpriority, shippriority) -DataSet> orders = getOrdersDataSet(env); -// get lineitem data set: (orderkey, extendedprice) -DataSet> lineitems = getLineitemDataSet(env); - -// orders filtered by year: (orderkey, custkey) -DataSet> ordersFilteredByYear = - // filter orders - orders.filter( - new FilterFunction>() { - @Override - public boolean filter(Tuple5 t) { - // status filter - if(!t.f1.equals(STATUS_FILTER)) { - return false; - // year filter - } else if(Integer.parseInt(t.f2.substring(0, 4)) <= YEAR_FILTER) { - return false; - // order priority filter - } else if(!t.f3.startsWith(OPRIO_FILTER)) { - return false; - } - return true; - } - }) - // project fields out that are no longer required - .project(0,4).types(Integer.class, Integer.class); - -// join orders with lineitems: (orderkey, shippriority, extendedprice) -DataSet> lineitemsOfOrders = - ordersFilteredByYear.joinWithHuge(lineitems) - .where(0).equalTo(0) - .projectFirst(0,1).projectSecond(1) - .types(Integer.class, Integer.class, Double.class); - -// extendedprice sums: (orderkey, shippriority, sum(extendedprice)) -DataSet> priceSums = - // group by order and sum extendedprice - lineitemsOfOrders.groupBy(0,1).aggregate(Aggregations.SUM, 2); - -// emit result -priceSums.writeAsCsv(outputPath); -~~~ - -The {% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/relational/TPCHQuery10.java "Relational Query program" %} implements the above query. It requires the following parameters to run: `--orders --lineitem --output `. - -
    -
    -Coming soon... - -The {% gh_link /flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala "Relational Query program" %} implements the above query. It requires the following parameters to run: `--orders --lineitem --output `. - -
    -
    - -The orders and lineitem files can be generated using the [TPC-H benchmark](http://www.tpc.org/tpch/) suite's data generator tool (DBGEN). -Take the following steps to generate arbitrary large input files for the provided Flink programs: - -1. Download and unpack DBGEN -2. Make a copy of *makefile.suite* called *Makefile* and perform the following changes: - -~~~bash -DATABASE = DB2 -MACHINE = LINUX -WORKLOAD = TPCH -CC = gcc -~~~ - -1. Build DBGEN using *make* -2. Generate lineitem and orders relations using dbgen. A scale factor - (-s) of 1 results in a generated data set with about 1 GB size. - -~~~bash -./dbgen -T o -s 1 -~~~ - {% top %} diff --git a/docs/dev/batch/fault_tolerance.md b/docs/dev/batch/fault_tolerance.md index f403791025700a..ee1e79731885e2 100644 --- a/docs/dev/batch/fault_tolerance.md +++ b/docs/dev/batch/fault_tolerance.md @@ -59,9 +59,9 @@ env.setNumberOfExecutionRetries(3) You can also define default values for the number of execution retries and the retry delay in the `flink-conf.yaml`: -~~~ +{% highlight yaml %} execution-retries.default: 3 -~~~ +{% endhighlight %} Retry Delays @@ -91,8 +91,8 @@ env.getConfig.setExecutionRetryDelay(5000) // 5000 milliseconds delay You can also define the default value for the retry delay in the `flink-conf.yaml`: -~~~ +{% highlight yaml %} execution-retries.delay: 10 s -~~~ +{% endhighlight %} {% top %} diff --git a/docs/dev/batch/hadoop_compatibility.md b/docs/dev/batch/hadoop_compatibility.md index 9f1478a243e17b..4e481822cf2af4 100644 --- a/docs/dev/batch/hadoop_compatibility.md +++ b/docs/dev/batch/hadoop_compatibility.md @@ -56,13 +56,13 @@ package. Add the following dependency to your `pom.xml` if you want to reuse Mappers and Reducers. -~~~xml +{% highlight xml %} org.apache.flink flink-hadoop-compatibility{{ site.scala_version_suffix }} {{site.version}} -~~~ +{% endhighlight %} ### Using Hadoop Data Types @@ -73,11 +73,14 @@ if you only want to use your Hadoop data types. See the ### Using Hadoop InputFormats -Hadoop input formats can be used to create a data source by using -one of the methods `readHadoopFile` or `createHadoopInput` of the -`ExecutionEnvironment`. The former is used for input formats derived +To use Hadoop `InputFormats` with Flink the format must first be wrapped +using either `readHadoopFile` or `createHadoopInput` of the +`HadoopInputs` utility class. +The former is used for input formats derived from `FileInputFormat` while the latter has to be used for general purpose input formats. +The resulting `InputFormat` can be used to create a data source by using +`ExecutionEnvironmen#createInput`. The resulting `DataSet` contains 2-tuples where the first field is the key and the second field is the value retrieved from the Hadoop @@ -88,28 +91,30 @@ The following example shows how to use Hadoop's `TextInputFormat`.
    -~~~java +{% highlight java %} ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet> input = - env.readHadoopFile(new TextInputFormat(), LongWritable.class, Text.class, textPath); + env.createInput(HadoopInputs.readHadoopFile(new TextInputFormat(), + LongWritable.class, Text.class, textPath)); // Do something with the data. [...] -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} val env = ExecutionEnvironment.getExecutionEnvironment val input: DataSet[(LongWritable, Text)] = - env.readHadoopFile(new TextInputFormat, classOf[LongWritable], classOf[Text], textPath) + env.createInput(HadoopInputs.readHadoopFile( + new TextInputFormat, classOf[LongWritable], classOf[Text], textPath)) // Do something with the data. [...] -~~~ +{% endhighlight %}
    @@ -128,7 +133,7 @@ The following example shows how to use Hadoop's `TextOutputFormat`.
    -~~~java +{% highlight java %} // Obtain the result we want to emit DataSet> hadoopResult = [...] @@ -144,12 +149,12 @@ TextOutputFormat.setOutputPath(job, new Path(outputPath)); // Emit data using the Hadoop TextOutputFormat. hadoopResult.output(hadoopOF); -~~~ +{% endhighlight %}
    -~~~scala +{% highlight scala %} // Obtain your result to emit. val hadoopResult: DataSet[(Text, IntWritable)] = [...] @@ -163,7 +168,7 @@ FileOutputFormat.setOutputPath(hadoopOF.getJobConf, new Path(resultPath)) hadoopResult.output(hadoopOF) -~~~ +{% endhighlight %}
    @@ -185,7 +190,7 @@ and can be used as regular Flink [FlatMapFunctions](dataset_transformations.html The following example shows how to use Hadoop `Mapper` and `Reducer` functions. -~~~java +{% highlight java %} // Obtain data to process somehow. DataSet> text = [...] @@ -199,7 +204,7 @@ DataSet> result = text .reduceGroup(new HadoopReduceCombineFunction( new Counter(), new Counter() )); -~~~ +{% endhighlight %} **Please note:** The Reducer wrapper works on groups as defined by Flink's [groupBy()](dataset_transformations.html#transformations-on-grouped-dataset) operation. It does not consider any custom partitioners, sort or grouping comparators you might have set in the `JobConf`. @@ -207,7 +212,7 @@ DataSet> result = text The following example shows a complete WordCount implementation using Hadoop data types, Input- and OutputFormats, and Mapper and Reducer implementations. -~~~java +{% highlight java %} ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // Set up the Hadoop TextInputFormat. @@ -245,6 +250,6 @@ result.output(hadoopOF); // Execute Program env.execute("Hadoop WordCount"); -~~~ +{% endhighlight %} {% top %} diff --git a/docs/dev/batch/index.md b/docs/dev/batch/index.md index f0fab8bab6c77d..521f4624b1a961 100644 --- a/docs/dev/batch/index.md +++ b/docs/dev/batch/index.md @@ -288,7 +288,7 @@ result = input1.join(input2) describe whether the join happens through partitioning or broadcasting, and whether it uses a sort-based or a hash-based algorithm. Please refer to the Transformations Guide for - a list of possible hints and an example.
    + a list of possible hints and an example.
    If no hint is specified, the system will try to make an estimate of the input sizes and pick the best strategy according to those estimates. {% highlight java %} @@ -401,12 +401,14 @@ DataSet result = in.partitionByRange(0) Custom Partitioning -

    Manually specify a partitioning over the data. +

    Assigns records based on a key to a specific partition using a custom Partitioner function. + The key can be specified as position key, expression key, and key selector function.
    - Note: This method works only on single field keys.

    + Note: This method only works with a single field key.

    {% highlight java %} DataSet> in = // [...] -DataSet result = in.partitionCustom(Partitioner partitioner, key) +DataSet result = in.partitionCustom(partitioner, key) + .mapPartition(new PartitionMapper()); {% endhighlight %} @@ -592,7 +594,7 @@ val output: DataSet[(Int, String, Double)] = input.sum(0).min(2) - + Join Joins two data sets by creating all pairs of elements that are equal on their keys. @@ -608,7 +610,7 @@ val result = input1.join(input2).where(0).equalTo(1) describe whether the join happens through partitioning or broadcasting, and whether it uses a sort-based or a hash-based algorithm. Please refer to the Transformations Guide for - a list of possible hints and an example.
    + a list of possible hints and an example.
    If no hint is specified, the system will try to make an estimate of the input sizes and pick the best strategy according to those estimates. {% highlight scala %} @@ -700,17 +702,17 @@ val result = in.partitionByRange(0).mapPartition { ... } {% endhighlight %} - Custom Partitioning -

    Manually specify a partitioning over the data. +

    Assigns records based on a key to a specific partition using a custom Partitioner function. + The key can be specified as position key, expression key, and key selector function.
    - Note: This method works only on single field keys.

    + Note: This method only works with a single field key.

    {% highlight scala %} val in: DataSet[(Int, String)] = // [...] val result = in - .partitionCustom(partitioner: Partitioner[K], key) + .partitionCustom(partitioner, key).mapPartition { ... } {% endhighlight %} @@ -824,12 +826,6 @@ File-based: - `readFileOfPrimitives(path, delimiter, Class)` / `PrimitiveInputFormat` - Parses files of new-line (or another char sequence) delimited primitive data types such as `String` or `Integer` using the given delimiter. -- `readHadoopFile(FileInputFormat, Key, Value, path)` / `FileInputFormat` - Creates a JobConf and reads file from the specified - path with the specified FileInputFormat, Key class and Value class and returns them as Tuple2. - -- `readSequenceFile(Key, Value, path)` / `SequenceFileInputFormat` - Creates a JobConf and reads file from the specified path with - type SequenceFileInputFormat, Key class and Value class and returns them as Tuple2. - Collection-based: @@ -878,14 +874,9 @@ DataSet> csvInput = env.readCsvFile("hdfs:///the/CSV/file DataSet> csvInput = env.readCsvFile("hdfs:///the/CSV/file") .pojoType(Person.class, "name", "age", "zipcode"); - -// read a file from the specified path of type TextInputFormat -DataSet> tuples = - env.readHadoopFile(new TextInputFormat(), LongWritable.class, Text.class, "hdfs://nnHost:nnPort/path/to/file"); - // read a file from the specified path of type SequenceFileInputFormat DataSet> tuples = - env.readSequenceFile(IntWritable.class, Text.class, "hdfs://nnHost:nnPort/path/to/file"); + env.createInput(HadoopInputs.readSequenceFile(IntWritable.class, Text.class, "hdfs://nnHost:nnPort/path/to/file")); // creates a set from some given elements DataSet value = env.fromElements("Foo", "bar", "foobar", "fubar"); @@ -974,9 +965,6 @@ File-based: - `readFileOfPrimitives(path, delimiter)` / `PrimitiveInputFormat` - Parses files of new-line (or another char sequence) delimited primitive data types such as `String` or `Integer` using the given delimiter. -- `readHadoopFile(FileInputFormat, Key, Value, path)` / `FileInputFormat` - Creates a JobConf and reads file from the specified - path with the specified FileInputFormat, Key class and Value class and returns them as Tuple2. - - `readSequenceFile(Key, Value, path)` / `SequenceFileInputFormat` - Creates a JobConf and reads file from the specified path with type SequenceFileInputFormat, Key class and Value class and returns them as Tuple2. @@ -1039,10 +1027,6 @@ val values = env.fromElements("Foo", "bar", "foobar", "fubar") // generate a number sequence val numbers = env.generateSequence(1, 10000000) -// read a file from the specified path of type TextInputFormat -val tuples = env.readHadoopFile(new TextInputFormat, classOf[LongWritable], - classOf[Text], "hdfs://nnHost:nnPort/path/to/file") - // read a file from the specified path of type SequenceFileInputFormat val tuples = env.readSequenceFile(classOf[IntWritable], classOf[Text], "hdfs://nnHost:nnPort/path/to/file") @@ -1630,7 +1614,7 @@ In object-reuse enabled mode, Flink's runtime minimizes the number of object ins Emitting Input Objects - You must not emit input objects, except for input objects of MapFunction, FlatMapFunction, MapPartitionFunction, GroupReduceFunction, GroupCombineFunction, CoGroupFunction, and InputFormat.next(reuse). + You must not emit input objects, except for input objects of MapFunction, FlatMapFunction, MapPartitionFunction, GroupReduceFunction, GroupCombineFunction, CoGroupFunction, and InputFormat.next(reuse). diff --git a/docs/dev/batch/iterations.md b/docs/dev/batch/iterations.md index f5b65ac8d79b48..be0ebb96480e11 100644 --- a/docs/dev/batch/iterations.md +++ b/docs/dev/batch/iterations.md @@ -102,7 +102,7 @@ There are multiple options to specify **termination conditions** for an iteratio You can also think about the iterate operator in pseudo-code: -~~~java +{% highlight java %} IterationState state = getInitialState(); while (!terminationCriterion()) { @@ -110,7 +110,7 @@ while (!terminationCriterion()) { } setFinalState(state); -~~~ +{% endhighlight %}
    @@ -130,14 +130,14 @@ In the following example, we **iteratively increment a set numbers**: 3. **Next Partial Solution**: The output of the step function will be the output of the map operator, i.e. records with incremented integers. 4. **Iteration Result**: After ten iterations, the initial numbers will have been incremented ten times, resulting in integers `11` to `15`. -~~~ +{% highlight plain %} // 1st 2nd 10th map(1) -> 2 map(2) -> 3 ... map(10) -> 11 map(2) -> 3 map(3) -> 4 ... map(11) -> 12 map(3) -> 4 map(4) -> 5 ... map(12) -> 13 map(4) -> 5 map(5) -> 6 ... map(13) -> 14 map(5) -> 6 map(6) -> 7 ... map(14) -> 15 -~~~ +{% endhighlight %} Note that **1**, **2**, and **4** can be arbitrary data flows. @@ -162,7 +162,7 @@ The default **termination condition** for delta iterations is specified by the * You can also think about the iterate operator in pseudo-code: -~~~java +{% highlight java %} IterationState workset = getInitialState(); IterationState solution = getInitialSolution(); @@ -173,7 +173,7 @@ while (!terminationCriterion()) { } setFinalState(solution); -~~~ +{% endhighlight %}
    diff --git a/docs/dev/batch/python.md b/docs/dev/batch/python.md index 22111027ee88e5..486aa18c9b768d 100644 --- a/docs/dev/batch/python.md +++ b/docs/dev/batch/python.md @@ -136,10 +136,7 @@ The first two do as the name suggests. Please refer to [Data Sinks](#data-sinks) for more information on writing to files. Once you specified the complete program you need to call `execute` on -the `Environment`. This will either execute on your local machine or submit your program -for execution on a cluster, depending on how Flink was started. You can force -a local execution by using `execute(local=True)`. - +the `Environment`. This will submit your program for execution on a cluster. {% top %} Project setup @@ -159,8 +156,7 @@ Lazy Evaluation All Flink programs are executed lazily: When the program's main method is executed, the data loading and transformations do not happen directly. Rather, each operation is created and added to the program's plan. The operations are actually executed when one of the `execute()` methods is invoked -on the Environment object. Whether the program is executed locally or on a cluster depends -on the environment of the program. +on the Environment object. The lazy evaluation lets you construct sophisticated programs that Flink executes as one holistically planned unit. diff --git a/docs/dev/best_practices.md b/docs/dev/best_practices.md index 24e7091974d9af..daf4aaf73d946f 100644 --- a/docs/dev/best_practices.md +++ b/docs/dev/best_practices.md @@ -149,20 +149,20 @@ Also, POJOs can be used to give large `Tuple`-types a name. Instead of using: -~~~java +{% highlight java %} Tuple11 var = new ...; -~~~ +{% endhighlight %} It is much easier to create a custom type extending from the large Tuple type. -~~~java +{% highlight java %} CustomType var = new ...; public static class CustomType extends Tuple11 { // constructor matching super } -~~~ +{% endhighlight %} ## Using Logback instead of Log4j @@ -192,7 +192,7 @@ public class MyClass implements MapFunction { In all cases were classes are executed with a classpath created by a dependency manager such as Maven, Flink will pull log4j into the classpath. -Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](../quickstart/java_api_quickstart.html). +Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](./projectsetup/java_api_quickstart.html). Change your projects `pom.xml` file like this: diff --git a/docs/dev/cluster_execution.md b/docs/dev/cluster_execution.md index f1d84e1b67b1fe..2b290549ff76d9 100644 --- a/docs/dev/cluster_execution.md +++ b/docs/dev/cluster_execution.md @@ -47,22 +47,22 @@ execute the program. If you are developing your program as a Maven project, you have to add the `flink-clients` module using this dependency: -~~~xml +{% highlight xml %} org.apache.flink flink-clients{{ site.scala_version_suffix }} {{ site.version }} -~~~ +{% endhighlight %} ### Example The following illustrates the use of the `RemoteEnvironment`: -~~~java +{% highlight java %} public static void main(String[] args) throws Exception { ExecutionEnvironment env = ExecutionEnvironment - .createRemoteEnvironment("flink-master", 6123, "/home/user/udfs.jar"); + .createRemoteEnvironment("flink-master", 8081, "/home/user/udfs.jar"); DataSet data = env.readTextFile("hdfs://path/to/file"); @@ -76,7 +76,7 @@ public static void main(String[] args) throws Exception { env.execute(); } -~~~ +{% endhighlight %} Note that the program contains custom user code and hence requires a JAR file with the classes of the code attached. The constructor of the remote environment diff --git a/docs/dev/connectors/cassandra.md b/docs/dev/connectors/cassandra.md index 6e1470a4367ad1..292314dafd23d1 100644 --- a/docs/dev/connectors/cassandra.md +++ b/docs/dev/connectors/cassandra.md @@ -72,10 +72,16 @@ The following configuration methods can be used: 4. _setMapperOptions(MapperOptions options)_ * Sets the mapper options that are used to configure the DataStax ObjectMapper. * Only applies when processing __POJO__ data types. -5. _enableWriteAheadLog([CheckpointCommitter committer])_ +5. _setMaxConcurrentRequests(int maxConcurrentRequests, Duration timeout)_ + * Sets the maximum allowed number of concurrent requests with a timeout for acquiring permits to execute. + * Only applies when __enableWriteAheadLog()__ is not configured. +6. _enableWriteAheadLog([CheckpointCommitter committer])_ * An __optional__ setting * Allows exactly-once processing for non-deterministic algorithms. -6. _build()_ +7. _setFailureHandler([CassandraFailureHandler failureHandler])_ + * An __optional__ setting + * Sets the custom failure handler. +8. _build()_ * Finalizes the configuration and constructs the CassandraSink instance. ### Write-ahead Log diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md index 52d1b58bf514d0..d0a159b254e034 100644 --- a/docs/dev/connectors/elasticsearch.md +++ b/docs/dev/connectors/elasticsearch.md @@ -55,6 +55,11 @@ of the Elasticsearch installation: 1.3.0 5.x + + flink-connector-elasticsearch6{{ site.scala_version_suffix }} + 1.6.0 + 6 and later versions + @@ -71,7 +76,7 @@ creating an `ElasticsearchSink` for requesting document actions against your clu ## Elasticsearch Sink -The `ElasticsearchSink` uses a `TransportClient` to communicate with an +The `ElasticsearchSink` uses a `TransportClient` (before 6.x) or `RestHighLevelClient` (starting with 6.x) to communicate with an Elasticsearch cluster. The example below shows how to configure and create a sink: @@ -79,6 +84,23 @@ The example below shows how to configure and create a sink:
    {% highlight java %} +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSink; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + DataStream input = ...; Map config = new HashMap<>(); @@ -110,6 +132,22 @@ input.addSink(new ElasticsearchSink<>(config, transportAddresses, new Elasticsea
    {% highlight java %} +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; +import org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + DataStream input = ...; Map config = new HashMap<>(); @@ -138,8 +176,86 @@ input.addSink(new ElasticsearchSink<>(config, transportAddresses, new Elasticsea } }));{% endhighlight %}
    +
    +{% highlight java %} +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; +import org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +DataStream input = ...; + +List httpHosts = new ArrayList<>(); +httpHosts.add(new HttpHost("127.0.0.1", 9200, "http")); +httpHosts.add(new HttpHost("10.2.3.1", 9200, "http")); + +// use a ElasticsearchSink.Builder to create an ElasticsearchSink +ElasticsearchSink.Builder esSinkBuilder = new ElasticsearchSink.Builder<>( + httpHosts, + new ElasticsearchSinkFunction() { + public IndexRequest createIndexRequest(String element) { + Map json = new HashMap<>(); + json.put("data", element); + + return Requests.indexRequest() + .index("my-index") + .type("my-type") + .source(json); + } + + @Override + public void process(String element, RuntimeContext ctx, RequestIndexer indexer) { + indexer.add(createIndexRequest(element)); + } + } +); + +// configuration for the bulk requests; this instructs the sink to emit after every element, otherwise they would be buffered +esSinkBuilder.setBulkFlushMaxActions(1); + +// provide a RestClientFactory for custom configuration on the internally created REST client +esSinkBuilder.setRestClientFactory( + restClientBuilder -> { + restClientBuilder.setDefaultHeaders(...) + restClientBuilder.setMaxRetryTimeoutMillis(...) + restClientBuilder.setPathPrefix(...) + restClientBuilder.setHttpClientConfigCallback(...) + } +); + +// finally, build and add the sink to the job's pipeline +input.addSink(esSinkBuilder.build()); +{% endhighlight %} +
    {% highlight scala %} +import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.streaming.api.datastream.DataStream +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSink +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer + +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.Requests +import org.elasticsearch.common.transport.InetSocketTransportAddress +import org.elasticsearch.common.transport.TransportAddress + +import java.net.InetAddress +import java.util.ArrayList +import java.util.HashMap +import java.util.List +import java.util.Map + val input: DataStream[String] = ... val config = new java.util.HashMap[String, String] @@ -166,6 +282,22 @@ input.addSink(new ElasticsearchSink(config, transportAddresses, new Elasticsearc
    {% highlight scala %} +import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.streaming.api.datastream.DataStream +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer +import org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink + +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.Requests + +import java.net.InetAddress +import java.net.InetSocketAddress +import java.util.ArrayList +import java.util.HashMap +import java.util.List +import java.util.Map + val input: DataStream[String] = ... val config = new java.util.HashMap[String, String] @@ -190,14 +322,75 @@ input.addSink(new ElasticsearchSink(config, transportAddresses, new Elasticsearc })) {% endhighlight %}
    +
    +{% highlight scala %} +import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.streaming.api.datastream.DataStream +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer +import org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink + +import org.apache.http.HttpHost +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.Requests + +import java.util.ArrayList +import java.util.List + +val input: DataStream[String] = ... + +val httpHosts = new java.util.ArrayList[HttpHost] +httpHosts.add(new HttpHost("127.0.0.1", 9200, "http")) +httpHosts.add(new HttpHost("10.2.3.1", 9200, "http")) + +val esSinkBuilder = new ElasticsearchSink.Builer[String]( + httpHosts, + new ElasticsearchSinkFunction[String] { + def createIndexRequest(element: String): IndexRequest = { + val json = new java.util.HashMap[String, String] + json.put("data", element) + + return Requests.indexRequest() + .index("my-index") + .type("my-type") + .source(json) + } + } +) + +// configuration for the bulk requests; this instructs the sink to emit after every element, otherwise they would be buffered +esSinkBuilder.setBulkFlushMaxActions(1) + +// provide a RestClientFactory for custom configuration on the internally created REST client +esSinkBuilder.setRestClientFactory( + restClientBuilder -> { + restClientBuilder.setDefaultHeaders(...) + restClientBuilder.setMaxRetryTimeoutMillis(...) + restClientBuilder.setPathPrefix(...) + restClientBuilder.setHttpClientConfigCallback(...) + } +) + +// finally, build and add the sink to the job's pipeline +input.addSink(esSinkBuilder.build) +{% endhighlight %} +
    -Note how a `Map` of `String`s is used to configure the `ElasticsearchSink`. +For Elasticsearch versions that still uses the now deprecated `TransportClient` to communicate +with the Elasticsearch cluster (i.e., versions equal or below 5.x), note how a `Map` of `String`s +is used to configure the `ElasticsearchSink`. This config map will be directly +forwarded when creating the internally used `TransportClient`. The configuration keys are documented in the Elasticsearch documentation [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html). Especially important is the `cluster.name` parameter that must correspond to the name of your cluster. +For Elasticsearch 6.x and above, internally, the `RestHighLevelClient` is used for cluster communication. +By default, the connector uses the default configurations for the REST client. To have custom +configuration for the REST client, users can provide a `RestClientFactory` implementation when +setting up the `ElasticsearchClient.Builder` that builds the sink. + Also note that the example only demonstrates performing a single index request for each incoming element. Generally, the `ElasticsearchSinkFunction` can be used to perform multiple requests of different types (ex., diff --git a/docs/dev/connectors/filesystem_sink.md b/docs/dev/connectors/filesystem_sink.md index 4e1f68a9f3abcb..79ed08e9d4129f 100644 --- a/docs/dev/connectors/filesystem_sink.md +++ b/docs/dev/connectors/filesystem_sink.md @@ -70,7 +70,8 @@ stored. The sink can be further configured by specifying a custom bucketer, writ By default the bucketing sink will split by the current system time when elements arrive and will use the datetime pattern `"yyyy-MM-dd--HH"` to name the buckets. This pattern is passed to -`SimpleDateFormat` with the current system time to form a bucket path. A new bucket will be created +`DateTimeFormatter` with the current system time and JVM's default timezone to form a bucket path. +Users can also specify a timezone for the bucketer to format bucket path. A new bucket will be created whenever a new date is encountered. For example, if you have a pattern that contains minutes as the finest granularity you will get a new bucket every minute. Each bucket is itself a directory that contains several part files: each parallel instance of the sink will create its own part file and @@ -89,8 +90,13 @@ and write them to part files, separated by newline. To specify a custom writer u on a `BucketingSink`. If you want to write Hadoop SequenceFiles you can use the provided `SequenceFileWriter` which can also be configured to use compression. -The last configuration option is the batch size. This specifies when a part file should be closed -and a new one started. (The default part file size is 384 MB). +There are two configuration options that specify when a part file should be closed +and a new one started: + +* By setting a batch size (The default part file size is 384 MB) +* By setting a batch roll over time interval (The default roll over interval is `Long.MAX_VALUE`) + +A new part file is started when either of these two conditions is satisfied. Example: @@ -100,9 +106,10 @@ Example: DataStream> input = ...; BucketingSink sink = new BucketingSink("/base/path"); -sink.setBucketer(new DateTimeBucketer("yyyy-MM-dd--HHmm")); +sink.setBucketer(new DateTimeBucketer("yyyy-MM-dd--HHmm", ZoneId.of("America/Los_Angeles"))); sink.setWriter(new SequenceFileWriter()); sink.setBatchSize(1024 * 1024 * 400); // this is 400 MB, +sink.setBatchRolloverInterval(20 * 60 * 1000); // this is 20 mins input.addSink(sink); @@ -113,9 +120,10 @@ input.addSink(sink); val input: DataStream[Tuple2[IntWritable, Text]] = ... val sink = new BucketingSink[String]("/base/path") -sink.setBucketer(new DateTimeBucketer[String]("yyyy-MM-dd--HHmm")) +sink.setBucketer(new DateTimeBucketer[String]("yyyy-MM-dd--HHmm", ZoneId.of("America/Los_Angeles"))) sink.setWriter(new SequenceFileWriter[IntWritable, Text]()) sink.setBatchSize(1024 * 1024 * 400) // this is 400 MB, +sink.setBatchRolloverInterval(20 * 60 * 1000); // this is 20 mins input.addSink(sink) @@ -125,13 +133,13 @@ input.addSink(sink) This will create a sink that writes to bucket files that follow this schema: -``` +{% highlight plain %} /base/path/{date-time}/part-{parallel-task}-{count} -``` +{% endhighlight %} Where `date-time` is the string that we get from the date/time format, `parallel-task` is the index -of the parallel sink instance and `count` is the running number of part files that where created -because of the batch size. +of the parallel sink instance and `count` is the running number of part files that were created +because of the batch size or batch roll over interval. For in-depth information, please refer to the JavaDoc for [BucketingSink](http://flink.apache.org/docs/latest/api/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.html). diff --git a/docs/dev/connectors/index.md b/docs/dev/connectors/index.md index 28ab2d34635223..b5405d4d06bebd 100644 --- a/docs/dev/connectors/index.md +++ b/docs/dev/connectors/index.md @@ -82,3 +82,5 @@ If the data involved has many fewer reads than writes, a better approach can be for an external application to pull from Flink the data it needs. The [Queryable State]({{ site.baseurl }}/dev/stream/state/queryable_state.html) interface enables this by allowing the state being managed by Flink to be queried on demand. + +{% top %} diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md index f28195c190f7c0..936d36b86e3b3b 100644 --- a/docs/dev/connectors/kafka.md +++ b/docs/dev/connectors/kafka.md @@ -73,13 +73,33 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is This connector supports Kafka messages with timestamps both for producing and consuming. - flink-connector-kafka-0.11_2.11 + flink-connector-kafka-0.11{{ site.scala_version_suffix }} 1.4.0 FlinkKafkaConsumer011
    FlinkKafkaProducer011 0.11.x Since 0.11.x Kafka does not support scala 2.10. This connector supports Kafka transactional messaging to provide exactly once semantic for the producer. + + flink-connector-kafka{{ site.scala_version_suffix }} + 1.7.0 + FlinkKafkaConsumer
    + FlinkKafkaProducer + >= 1.0.0 + + This universal Kafka connector attempts to track the latest version of the Kafka client. + The version of the client it uses may change between Flink releases. + Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later. + However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated + flink-connector-kafka-0.11{{ site.scala_version_suffix }} and flink-connector-kafka-0.10{{ site.scala_version_suffix }} respectively. +
    + Attention: as of Flink 1.7 the universal Kafka connector is considered to be + in a BETA status and might not be as stable as the 0.11 connector. + In case of problems with the universal connector, you can try to use flink-connector-kafka-0.11{{ site.scala_version_suffix }} + which should be compatible with all of the Kafka versions starting from 0.11. +
    + + @@ -88,21 +108,53 @@ Then, import the connector in your maven project: {% highlight xml %} org.apache.flink - flink-connector-kafka-0.8{{ site.scala_version_suffix }} - {{site.version }} + flink-connector-kafka{{ site.scala_version_suffix }} + {{ site.version }} {% endhighlight %} -Note that the streaming connectors are currently not part of the binary distribution. See how to link with them for cluster execution [here]({{ site.baseurl}}/dev/linking.html). +Note that the streaming connectors are currently not part of the binary distribution. +See how to link with them for cluster execution [here]({{ site.baseurl}}/dev/linking.html). ## Installing Apache Kafka * Follow the instructions from [Kafka's quickstart](https://kafka.apache.org/documentation.html#quickstart) to download the code and launch a server (launching a Zookeeper and a Kafka server is required every time before starting the application). * If the Kafka and Zookeeper servers are running on a remote machine, then the `advertised.host.name` setting in the `config/server.properties` file must be set to the machine's IP address. +## Kafka 1.0.0+ Connector + +Starting with Flink 1.7, there is a new universal Kafka connector that does not track a specific Kafka major version. +Rather, it tracks the latest version of Kafka at the time of the Flink release. + +If your Kafka broker version is 1.0.0 or newer, you should use this Kafka connector. +If you use an older version of Kafka (0.11, 0.10, 0.9, or 0.8), you should use the connector corresponding to the broker version. + +### Compatibility + +The universal Kafka connector is compatible with older and newer Kafka brokers through the compatibility guarantees of the Kafka client API and broker. +It is compatible with broker versions 0.11.0 or newer, depending on the features used. +For details on Kafka compatibility, please refer to the [Kafka documentation](https://kafka.apache.org/protocol.html#protocol_compatibility). + +### Usage + +To use the universal Kafka connector add a dependency to it: + +{% highlight xml %} + + org.apache.flink + flink-connector-kafka{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + +Then instantiate the new source (`FlinkKafkaConsumer`) and sink (`FlinkKafkaProducer`). +The API is backward compatible with the Kafka 0.11 connector, +except of dropping specific Kafka version from the module and class names. + ## Kafka Consumer -Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or `09` for Kafka 0.9.0.x versions, etc.). It provides access to one or more Kafka topics. +Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or 09 for Kafka 0.9.0.x versions, etc. +or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). It provides access to one or more Kafka topics. The constructor accepts the following arguments: @@ -166,6 +218,39 @@ For convenience, Flink provides the following schemas: The KeyValue objectNode contains a "key" and "value" field which contain all fields, as well as an optional "metadata" field that exposes the offset/partition/topic for this message. +3. `AvroDeserializationSchema` which reads data serialized with Avro format using a statically provided schema. It can + infer the schema from Avro generated classes (`AvroDeserializationSchema.forSpecific(...)`) or it can work with `GenericRecords` + with a manually provided schema (with `AvroDeserializationSchema.forGeneric(...)`). This deserialization schema expects that + the serialized records DO NOT contain embedded schema. + + - There is also a version of this schema available that can lookup the writer's schema (schema which was used to write the record) in + [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html). Using these deserialization schema + record will be read with the schema that was retrieved from Schema Registry and transformed to a statically provided( either through + `ConfluentRegistryAvroDeserializationSchema.forGeneric(...)` or `ConfluentRegistryAvroDeserializationSchema.forSpecific(...)`). + +
    To use this deserialization schema one has to add the following additional dependency: + +
    +
    +{% highlight xml %} + + org.apache.flink + flink-avro + {{site.version }} + +{% endhighlight %} +
    +
    +{% highlight xml %} + + org.apache.flink + flink-avro-confluent-registry + {{site.version }} + +{% endhighlight %} +
    +
    + When encountering a corrupted message that cannot be deserialized for any reason, there are two options - either throwing an exception from the `deserialize(...)` method which will cause the job to fail and be restarted, or returning `null` to allow @@ -191,6 +276,7 @@ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEn FlinkKafkaConsumer08 myConsumer = new FlinkKafkaConsumer08<>(...); myConsumer.setStartFromEarliest(); // start from the earliest record possible myConsumer.setStartFromLatest(); // start from the latest record +myConsumer.setStartFromTimestamp(...); // start from specified epoch timestamp (milliseconds) myConsumer.setStartFromGroupOffsets(); // the default behaviour DataStream stream = env.addSource(myConsumer); @@ -204,6 +290,7 @@ val env = StreamExecutionEnvironment.getExecutionEnvironment() val myConsumer = new FlinkKafkaConsumer08[String](...) myConsumer.setStartFromEarliest() // start from the earliest record possible myConsumer.setStartFromLatest() // start from the latest record +myConsumer.setStartFromTimestamp(...) // start from specified epoch timestamp (milliseconds) myConsumer.setStartFromGroupOffsets() // the default behaviour val stream = env.addSource(myConsumer) @@ -221,6 +308,11 @@ All versions of the Flink Kafka Consumer have the above explicit configuration m * `setStartFromEarliest()` / `setStartFromLatest()`: Start from the earliest / latest record. Under these modes, committed offsets in Kafka will be ignored and not used as starting positions. + * `setStartFromTimestamp(long)`: Start from the specified timestamp. For each partition, the record + whose timestamp is larger than or equal to the specified timestamp will be used as the start position. + If a partition's latest record is earlier than the timestamp, the partition will simply be read + from the latest record. Under this mode, committed offsets in Kafka will be ignored and not used as + starting positions. You can also specify the exact offsets the consumer should start from for each partition: @@ -444,10 +536,18 @@ the `Watermark getCurrentWatermark()` (for periodic) or the `Watermark checkAndGetNextWatermark(T lastElement, long extractedTimestamp)` (for punctuated) is called to determine if a new watermark should be emitted and with which timestamp. +**Note**: If a watermark assigner depends on records read from Kafka to advance its watermarks +(which is commonly the case), all topics and partitions need to have a continuous stream of records. +Otherwise, the watermarks of the whole application cannot advance and all time-based operations, +such as time windows or functions with timers, cannot make progress. A single idle Kafka partition causes this behavior. +A Flink improvement is planned to prevent this from happening +(see [FLINK-5479: Per-partition watermarks in FlinkKafkaConsumer should consider idle partitions]( +https://issues.apache.org/jira/browse/FLINK-5479)). +In the meanwhile, a possible workaround is to send *heartbeat messages* to all consumed partitions that advance the watermarks of idle partitions. ## Kafka Producer -Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc.). +Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaProducer` for Kafka >= 1.0.0 versions). It allows writing a stream of records to one or more Kafka topics. Example: @@ -560,26 +660,13 @@ we recommend setting the number of retries to a higher value. **Note**: There is currently no transactional producer for Kafka, so Flink can not guarantee exactly-once delivery into a Kafka topic. -
    - Attention: Depending on your Kafka configuration, even after Kafka acknowledges - writes you can still experience data loss. In particular keep in mind the following Kafka settings: -
      -
    • acks
      • -
    • log.flush.interval.messages
      • -
    • log.flush.interval.ms
      • -
    • log.flush.*
      • -
    - Default values for the above options can easily lead to data loss. Please refer to Kafka documentation - for more explanation. -
    - -#### Kafka 0.11 +#### Kafka 0.11 and newer -With Flink's checkpointing enabled, the `FlinkKafkaProducer011` can provide +With Flink's checkpointing enabled, the `FlinkKafkaProducer011` (`FlinkKafkaProducer` for Kafka >= 1.0.0 versions) can provide exactly-once delivery guarantees. Besides enabling Flink's checkpointing, you can also choose three different modes of operating -chosen by passing appropriate `semantic` parameter to the `FlinkKafkaProducer011`: +chosen by passing appropriate `semantic` parameter to the `FlinkKafkaProducer011` (`FlinkKafkaProducer` for Kafka >= 1.0.0 versions): * `Semantic.NONE`: Flink will not guarantee anything. Produced records can be lost or they can be duplicated. @@ -590,21 +677,6 @@ chosen by passing appropriate `semantic` parameter to the `FlinkKafkaProducer011 or `read_uncommitted` - the latter one is the default value) for any application consuming records from Kafka. -
    - Attention: Depending on your Kafka configuration, even after Kafka acknowledges - writes you can still experience data losses. In particular keep in mind about following properties - in Kafka config: -
      -
    • acks
      • -
    • log.flush.interval.messages
      • -
    • log.flush.interval.ms
      • -
    • log.flush.*
      • -
    - Default values for the above options can easily lead to data loss. Please refer to the Kafka documentation - for more explanation. -
    - - ##### Caveats `Semantic.EXACTLY_ONCE` mode relies on the ability to commit transactions @@ -652,7 +724,8 @@ application before first checkpoint completes, by factor larger then `FlinkKafka ## Using Kafka timestamps and Flink event time in Kafka 0.10 -Since Apache Kafka 0.10+, Kafka's messages can carry [timestamps](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message), indicating +Since Apache Kafka 0.10+, Kafka's messages can carry +[timestamps](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message), indicating the time the event has occurred (see ["event time" in Apache Flink](../event_time.html)) or the time when the message has been written to the Kafka broker. @@ -712,19 +785,56 @@ Flink provides first-class support through the Kafka connector to authenticate t configured for Kerberos. Simply configure Flink in `flink-conf.yaml` to enable Kerberos authentication for Kafka like so: 1. Configure Kerberos credentials by setting the following - - - `security.kerberos.login.use-ticket-cache`: By default, this is `true` and Flink will attempt to use Kerberos credentials in ticket caches managed by `kinit`. - Note that when using the Kafka connector in Flink jobs deployed on YARN, Kerberos authorization using ticket caches will not work. This is also the case when deploying using Mesos, as authorization using ticket cache is not supported for Mesos deployments. + - `security.kerberos.login.use-ticket-cache`: By default, this is `true` and Flink will attempt to use Kerberos credentials in ticket caches managed by `kinit`. + Note that when using the Kafka connector in Flink jobs deployed on YARN, Kerberos authorization using ticket caches will not work. + This is also the case when deploying using Mesos, as authorization using ticket cache is not supported for Mesos deployments. - `security.kerberos.login.keytab` and `security.kerberos.login.principal`: To use Kerberos keytabs instead, set values for both of these properties. 2. Append `KafkaClient` to `security.kerberos.login.contexts`: This tells Flink to provide the configured Kerberos credentials to the Kafka login context to be used for Kafka authentication. -Once Kerberos-based Flink security is enabled, you can authenticate to Kafka with either the Flink Kafka Consumer or Producer by simply including the following two settings in the provided properties configuration that is passed to the internal Kafka client: +Once Kerberos-based Flink security is enabled, you can authenticate to Kafka with either the Flink Kafka Consumer or Producer +by simply including the following two settings in the provided properties configuration that is passed to the internal Kafka client: - Set `security.protocol` to `SASL_PLAINTEXT` (default `NONE`): The protocol used to communicate to Kafka brokers. When using standalone Flink deployment, you can also use `SASL_SSL`; please see how to configure the Kafka client for SSL [here](https://kafka.apache.org/documentation/#security_configclients). -- Set `sasl.kerberos.service.name` to `kafka` (default `kafka`): The value for this should match the `sasl.kerberos.service.name` used for Kafka broker configurations. A mismatch in service name between client and server configuration will cause the authentication to fail. +- Set `sasl.kerberos.service.name` to `kafka` (default `kafka`): The value for this should match the `sasl.kerberos.service.name` used for Kafka broker configurations. +A mismatch in service name between client and server configuration will cause the authentication to fail. For more information on Flink configuration for Kerberos security, please see [here]({{ site.baseurl}}/ops/config.html). You can also find [here]({{ site.baseurl}}/ops/security-kerberos.html) further details on how Flink internally setups Kerberos-based security. +## Troubleshooting + +
    +If you have a problem with Kafka when using Flink, keep in mind that Flink only wraps +KafkaConsumer or +KafkaProducer +and your problem might be independent of Flink and sometimes can be solved by upgrading Kafka brokers, +reconfiguring Kafka brokers or reconfiguring KafkaConsumer or KafkaProducer in Flink. +Some examples of common problems are listed below. +
    + +### Data loss + +Depending on your Kafka configuration, even after Kafka acknowledges +writes you can still experience data loss. In particular keep in mind about the following properties +in Kafka config: + +- `acks` +- `log.flush.interval.messages` +- `log.flush.interval.ms` +- `log.flush.*` + +Default values for the above options can easily lead to data loss. +Please refer to the Kafka documentation for more explanation. + +### UnknownTopicOrPartitionException + +One possible cause of this error is when a new leader election is taking place, +for example after or during restarting a Kafka broker. +This is a retriable exception, so Flink job should be able to restart and resume normal operation. +It also can be circumvented by changing `retries` property in the producer settings. +However this might cause reordering of messages, +which in turn if undesired can be circumvented by setting `max.in.flight.requests.per.connection` to 1. + {% top %} diff --git a/docs/dev/connectors/kinesis.md b/docs/dev/connectors/kinesis.md index 4ee9c6b03d4636..5e0b2f3aa82605 100644 --- a/docs/dev/connectors/kinesis.md +++ b/docs/dev/connectors/kinesis.md @@ -52,6 +52,15 @@ cd flink-dist mvn clean install -Pinclude-kinesis -DskipTests {% endhighlight %} +Attention For Flink versions 1.4.2 and below, the KPL client version +used by default in the Kinesis connectors, KPL 0.12.5, is no longer supported by AWS Kinesis Streams +(see [here](https://docs.aws.amazon.com/streams/latest/dev/kinesis-kpl-upgrades.html)). +This means that when building the Kinesis connector, you will need to specify a higher version KPL client (above 0.12.6) +in order for the Flink Kinesis Producer to work. You can do this by specifying the preferred version via the +`aws.kinesis-kpl.version` property, like so: +{% highlight bash %} +mvn clean install -Pinclude-kinesis -Daws.kinesis-kpl.version=0.12.6 -DskipTests +{% endhighlight %} The streaming connectors are not part of the binary distribution. See how to link with them for cluster execution [here]({{site.baseurl}}/dev/linking.html). @@ -104,7 +113,7 @@ The above is a simple example of using the consumer. Configuration for the consu instance, the configuration keys for which can be found in `ConsumerConfigConstants`. The example demonstrates consuming a single Kinesis stream in the AWS region "us-east-1". The AWS credentials are supplied using the basic method in which the AWS access key ID and secret access key are directly supplied in the configuration (other options are setting -`ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER` to `ENV_VAR`, `SYS_PROP`, `PROFILE`, and `AUTO`). Also, data is being consumed +`ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER` to `ENV_VAR`, `SYS_PROP`, `PROFILE`, `ASSUME_ROLE`, and `AUTO`). Also, data is being consumed from the newest position in the Kinesis stream (the other option will be setting `ConsumerConfigConstants.STREAM_INITIAL_POSITION` to `TRIM_HORIZON`, which lets the consumer start reading the Kinesis stream from the earliest record possible). @@ -286,6 +295,8 @@ producerConfig.put("RecordTtl", "30000"); producerConfig.put("RequestTimeout", "6000"); producerConfig.put("ThreadPoolSize", "15"); +// Disable Aggregation if it's not supported by a consumer +// producerConfig.put("AggregationEnabled", "false"); // Switch KinesisProducer's threading model // producerConfig.put("ThreadingModel", "PER_REQUEST"); @@ -312,6 +323,8 @@ producerConfig.put("RecordTtl", "30000") producerConfig.put("RequestTimeout", "6000") producerConfig.put("ThreadPoolSize", "15") +// Disable Aggregation if it's not supported by a consumer +// producerConfig.put("AggregationEnabled", "false") // Switch KinesisProducer's threading model // producerConfig.put("ThreadingModel", "PER_REQUEST") @@ -340,6 +353,36 @@ Since Flink 1.4.0, `FlinkKinesisProducer` switches its default underlying KPL fr Users can still switch back to one-thread-per-request mode by setting a key-value pair of `ThreadingModel` and `PER_REQUEST` in `java.util.Properties`, as shown in the code commented out in above example. +### Backpressure + +By default, `FlinkKinesisProducer` does not backpressure. Instead, records that +cannot be sent because of the rate restriction of 1 MB per second per shard are +buffered in an unbounded queue and dropped when their `RecordTtl` expires. + +To avoid data loss, you can enable backpressuring by restricting the size of the +internal queue: + +``` +// 200 Bytes per record, 1 shard +kinesis.setQueueLimit(500); +``` + +The value for `queueLimit` depends on the expected record size. To choose a good +value, consider that Kinesis is rate-limited to 1MB per second per shard. If +less than one second's worth of records is buffered, then the queue may not be +able to operate at full capacity. With the default `RecordMaxBufferedTime` of +100ms, a queue size of 100kB per shard should be sufficient. The `queueLimit` +can then be computed via + +``` +queue limit = (number of shards * queue size per shard) / record size +``` + +E.g. for 200Bytes per record and 8 shards, a queue limit of 4000 is a good +starting point. If the queue size limits throughput (below 1MB per second per +shard), try increasing the queue limit slightly. + + ## Using Non-AWS Kinesis Endpoints for Testing It is sometimes desirable to have Flink operate as a consumer or producer against a non-AWS Kinesis endpoint such as diff --git a/docs/dev/connectors/streamfile_sink.md b/docs/dev/connectors/streamfile_sink.md new file mode 100644 index 00000000000000..82ab5620571ba4 --- /dev/null +++ b/docs/dev/connectors/streamfile_sink.md @@ -0,0 +1,126 @@ +--- +title: "Streaming File Sink" +nav-title: Streaming File Sink +nav-parent_id: connectors +nav-pos: 5 +--- + + +This connector provides a Sink that writes partitioned files to filesystems +supported by the [Flink `FileSystem` abstraction]({{ site.baseurl}}/ops/filesystems.html). + +Important Note: For S3, the `StreamingFileSink` +supports only the [Hadoop-based](https://hadoop.apache.org/) FileSystem implementation, not +the implementation based on [Presto](https://prestodb.io/). In case your job uses the +`StreamingFileSink` to write to S3 but you want to use the Presto-based one for checkpointing, +it is advised to use explicitly *"s3a://"* (for Hadoop) as the scheme for the target path of +the sink and *"s3p://"* for checkpointing (for Presto). Using *"s3://"* for both the sink +and checkpointing may lead to unpredictable behavior, as both implementations "listen" to that scheme. + +Since in streaming the input is potentially infinite, the streaming file sink writes data +into buckets. The bucketing behaviour is configurable but a useful default is time-based +bucketing where we start writing a new bucket every hour and thus get +individual files that each contain a part of the infinite output stream. + +Within a bucket, we further split the output into smaller part files based on a +rolling policy. This is useful to prevent individual bucket files from getting +too big. This is also configurable but the default policy rolls files based on +file size and a timeout, *i.e* if no new data was written to a part file. + +The `StreamingFileSink` supports both row-wise encoding formats and +bulk-encoding formats, such as [Apache Parquet](http://parquet.apache.org). + +#### Using Row-encoded Output Formats + +The only required configuration are the base path where we want to output our +data and an +[Encoder]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/api/common/serialization/Encoder.html) +that is used for serializing records to the `OutputStream` for each file. + +Basic usage thus looks like this: + + +
    +
    +{% highlight java %} +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; + +DataStream input = ...; + +final StreamingFileSink sink = StreamingFileSink + .forRowFormat(new Path(outputPath), new SimpleStringEncoder<>("UTF-8")) + .build(); + +input.addSink(sink); + +{% endhighlight %} +
    +
    +{% highlight scala %} +import org.apache.flink.api.common.serialization.SimpleStringEncoder +import org.apache.flink.core.fs.Path +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink + +val input: DataStream[String] = ... + +val sink: StreamingFileSink[String] = StreamingFileSink + .forRowFormat(new Path(outputPath), new SimpleStringEncoder[String]("UTF-8")) + .build() + +input.addSink(sink) + +{% endhighlight %} +
    +
    + +This will create a streaming sink that creates hourly buckets and uses a +default rolling policy. The default bucket assigner is +[DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) +and the default rolling policy is +[DefaultRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/DefaultRollingPolicy.html). +You can specify a custom +[BucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssigner.html) +and +[RollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicy.html) +on the sink builder. Please check out the JavaDoc for +[StreamingFileSink]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.html) +for more configuration options and more documentation about the workings and +interactions of bucket assigners and rolling policies. + +#### Using Bulk-encoded Output Formats + +In the above example we used an `Encoder` that can encode or serialize each +record individually. The streaming file sink also supports bulk-encoded output +formats such as [Apache Parquet](http://parquet.apache.org). To use these, +instead of `StreamingFileSink.forRowFormat()` you would use +`StreamingFileSink.forBulkFormat()` and specify a `BulkWriter.Factory`. + +[ParquetAvroWriters]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/parquet/avro/ParquetAvroWriters.html) +has static methods for creating a `BulkWriter.Factory` for various types. + +
    + IMPORTANT: Bulk-encoding formats can only be combined with the + `OnCheckpointRollingPolicy`, which rolls the in-progress part file on + every checkpoint. +
    + +{% top %} diff --git a/docs/dev/datastream_api.md b/docs/dev/datastream_api.md index 3cce5be1c01465..b2c8cb81c2c973 100644 --- a/docs/dev/datastream_api.md +++ b/docs/dev/datastream_api.md @@ -125,9 +125,9 @@ object WindowWordCount { To run the example program, start the input stream with netcat first from a terminal: -~~~bash +{% highlight bash %} nc -lk 9999 -~~~ +{% endhighlight %} Just type some words hitting return for a new word. These will be the input to the word count program. If you want to see counts greater than 1, type the same word again and again within @@ -192,7 +192,7 @@ Collection-based: Custom: -- `addSource` - Attache a new source function. For example, to read from Apache Kafka you can use +- `addSource` - Attach a new source function. For example, to read from Apache Kafka you can use `addSource(new FlinkKafkaConsumer08<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/index.html) for more details.
    @@ -453,9 +453,6 @@ The `StreamExecutionEnvironment` contains the `ExecutionConfig` which allows to Please refer to [execution configuration]({{ site.baseurl }}/dev/execution_configuration.html) for an explanation of most parameters. These parameters pertain specifically to the DataStream API: -- `enableTimestamps()` / **`disableTimestamps()`**: Attach a timestamp to each event emitted from a source. - `areTimestampsEnabled()` returns the current value. - - `setAutoWatermarkInterval(long milliseconds)`: Set the interval for automatic watermark emission. You can get the current value with `long getAutoWatermarkInterval()` diff --git a/docs/dev/event_time.md b/docs/dev/event_time.md index a3e697d8f7e6b8..1d747aa2c33a32 100644 --- a/docs/dev/event_time.md +++ b/docs/dev/event_time.md @@ -35,31 +35,34 @@ Flink supports different notions of *time* in streaming programs. respective operation. When a streaming program runs on processing time, all time-based operations (like time windows) will - use the system clock of the machines that run the respective operator. For example, an hourly + use the system clock of the machines that run the respective operator. An hourly processing time window will include all records that arrived at a specific operator between the - times when the system clock indicated the full hour. + times when the system clock indicated the full hour. For example, if an application + begins running at 9:15am, the first hourly processing time window will include events + processed between 9:15am and 10:00am, the next window will include events processed between 10:00am and 11:00am, and so on. Processing time is the simplest notion of time and requires no coordination between streams and machines. It provides the best performance and the lowest latency. However, in distributed and asynchronous environments processing time does not provide determinism, because it is susceptible to the speed at which - records arrive in the system (for example from the message queue), and to the speed at which the - records flow between operators inside the system. + records arrive in the system (for example from the message queue), to the speed at which the + records flow between operators inside the system, and to outages (scheduled, or otherwise). - **Event time:** Event time is the time that each individual event occurred on its producing device. - This time is typically embedded within the records before they enter Flink and that *event timestamp* - can be extracted from the record. An hourly event time window will contain all records that carry an - event timestamp that falls into that hour, regardless of when the records arrive, and in what order - they arrive. - - Event time gives correct results even on out-of-order events, late events, or on replays - of data from backups or persistent logs. In event time, the progress of time depends on the data, + This time is typically embedded within the records before they enter Flink, and that *event timestamp* + can be extracted from each record. In event time, the progress of time depends on the data, not on any wall clocks. Event time programs must specify how to generate *Event Time Watermarks*, - which is the mechanism that signals progress in event time. The mechanism is - described below. + which is the mechanism that signals progress in event time. This watermarking mechanism is + described in a later section, [below](#event-time-and-watermarks). + + In a perfect world, event time processing would yield completely consistent and deterministic results, regardless of when events arrive, or their ordering. + However, unless the events are known to arrive in-order (by timestamp), event time processing incurs some latency while waiting for out-of-order events. As it is only possible to wait for a finite period of time, this places a limit on how deterministic event time applications can be. + + Assuming all of the data has arrived, event time operations will behave as expected, and produce correct and consistent results even when working with out-of-order or late events, or when reprocessing historic data. For example, an hourly event time window will contain all records + that carry an event timestamp that falls into that hour, regardless of the order in which they arrive, or when they are processed. (See the section on [late events](#late-elements) for more information.) + - Event time processing often incurs a certain latency, due to its nature of waiting a certain time for - late events and out-of-order events. Because of that, event time programs are often combined with - *processing time* operations. + + Note that sometimes when event time programs are processing live data in real-time, they will use some *processing time* operations in order to guarantee that they are progressing in a timely fashion. - **Ingestion time:** Ingestion time is the time that events enter Flink. At the source operator each record gets the source's current time as a timestamp, and time-based operations (like time windows) @@ -177,6 +180,8 @@ Once a watermark reaches an operator, the operator can advance its internal *eve A data stream with events (out of order) and watermarks +Note that event time is inherited by a freshly created stream element (or elements) from either the event that produced them or +from watermark that triggered creation of those elements. ## Watermarks in Parallel Streams @@ -194,6 +199,8 @@ The figure below shows an example of events and watermarks flowing through paral Parallel data streams and operators with events and watermarks +Note that the Kafka source supports per-partition watermarking, which you can read more about [here]({{ site.baseurl }}/dev/event_timestamps_watermarks.html#timestamps-per-kafka-partition). + ## Late Elements @@ -201,17 +208,43 @@ It is possible that certain elements will violate the watermark condition, meani more elements with timestamp *t' <= t* will occur. In fact, in many real world setups, certain elements can be arbitrarily delayed, making it impossible to specify a time by which all elements of a certain event timestamp will have occurred. Furthermore, even if the lateness can be bounded, delaying the watermarks by too much is often not desirable, because it -causes too much delay in the evaluation of the event time windows. +causes too much delay in the evaluation of event time windows. For this reason, streaming programs may explicitly expect some *late* elements. Late elements are elements that arrive after the system's event time clock (as signaled by the watermarks) has already passed the time of the late element's timestamp. See [Allowed Lateness]({{ site.baseurl }}/dev/stream/operators/windows.html#allowed-lateness) for more information on how to work with late elements in event time windows. +## Idling sources + +Currently, with pure event time watermarks generators, watermarks can not progress if there are no elements +to be processed. That means in case of gap in the incoming data, event time will not progress and for +example the window operator will not be triggered and thus existing windows will not be able to produce any +output data. + +To circumvent this one can use periodic watermark assigners that don't only assign based on +element timestamps. An example solution could be an assigner that switches to using current processing time +as the time basis after not observing new events for a while. + +Sources can be marked as idle using `SourceFunction.SourceContext#markAsTemporarilyIdle`. For details please refer to the Javadoc of +this method as well as `StreamStatus`. ## Debugging Watermarks Please refer to the [Debugging Windows & Event Time]({{ site.baseurl }}/monitoring/debugging_event_time.html) section for debugging watermarks at runtime. +## How operators are processing watermarks + +As a general rule, operators are required to completely process a given watermark before forwarding it downstream. For example, +`WindowOperator` will first evaluate which windows should be fired, and only after producing all of the output triggered by +the watermark will the watermark itself be sent downstream. In other words, all elements produced due to occurrence of a watermark +will be emitted before the watermark. + +The same rule applies to `TwoInputStreamOperator`. However, in this case the current watermark of the operator is defined as +the minimum of both of its inputs. + +The details of this behavior are defined by the implementations of the `OneInputStreamOperator#processWatermark`, +`TwoInputStreamOperator#processWatermark1` and `TwoInputStreamOperator#processWatermark2` methods. + {% top %} diff --git a/docs/dev/event_timestamps_watermarks.md b/docs/dev/event_timestamps_watermarks.md index acde9e48254d09..cb1c5d4e0210b8 100644 --- a/docs/dev/event_timestamps_watermarks.md +++ b/docs/dev/event_timestamps_watermarks.md @@ -58,14 +58,14 @@ progress in event time. There are two ways to assign timestamps and generate watermarks: 1. Directly in the data stream source - 2. Via a timestamp assigner / watermark generator: in Flink timestamp assigners also define the watermarks to be emitted + 2. Via a timestamp assigner / watermark generator: in Flink, timestamp assigners also define the watermarks to be emitted Attention Both timestamps and watermarks are specified as milliseconds since the Java epoch of 1970-01-01T00:00:00Z. ### Source Functions with Timestamps and Watermarks -Stream sources can also directly assign timestamps to the elements they produce, and they can also emit watermarks. +Stream sources can directly assign timestamps to the elements they produce, and they can also emit watermarks. When this is done, no timestamp assigner is needed. Note that if a timestamp assigner is used, any timestamps and watermarks provided by the source will be overwritten. @@ -180,7 +180,7 @@ The interval (every *n* milliseconds) in which the watermark will be generated i called each time, and a new watermark will be emitted if the returned watermark is non-null and larger than the previous watermark. -Two simple examples of timestamp assigners with periodic watermark generation are below. +Here we show two simple examples of timestamp assigners that use periodic watermark generation. Note that Flink ships with a `BoundedOutOfOrdernessTimestampExtractor` similar to the `BoundedOutOfOrdernessGenerator` shown below, which you can read about [here]({{ site.baseurl }}/dev/event_timestamp_extractors.html#assigners-allowing-a-fixed-amount-of-lateness).
    @@ -190,7 +190,7 @@ Two simple examples of timestamp assigners with periodic watermark generation ar * but only to a certain degree. The latest elements for a certain timestamp t will arrive * at most n milliseconds after the earliest elements for timestamp t. */ -public class BoundedOutOfOrdernessGenerator extends AssignerWithPeriodicWatermarks { +public class BoundedOutOfOrdernessGenerator implements AssignerWithPeriodicWatermarks { private final long maxOutOfOrderness = 3500; // 3.5 seconds @@ -214,7 +214,7 @@ public class BoundedOutOfOrdernessGenerator extends AssignerWithPeriodicWatermar * This generator generates watermarks that are lagging behind processing time by a fixed amount. * It assumes that elements arrive in Flink after a bounded delay. */ -public class TimeLagWatermarkGenerator extends AssignerWithPeriodicWatermarks { +public class TimeLagWatermarkGenerator implements AssignerWithPeriodicWatermarks { private final long maxTimeLag = 5000; // 5 seconds @@ -242,7 +242,7 @@ class BoundedOutOfOrdernessGenerator extends AssignerWithPeriodicWatermarks[MyEv val maxOutOfOrderness = 3500L // 3.5 seconds - var currentMaxTimestamp: Long + var currentMaxTimestamp: Long = _ override def extractTimestamp(element: MyEvent, previousElementTimestamp: Long): Long = { val timestamp = element.getCreationTime() @@ -292,7 +292,7 @@ new watermark will be emitted.
    {% highlight java %} -public class PunctuatedAssigner extends AssignerWithPunctuatedWatermarks { +public class PunctuatedAssigner implements AssignerWithPunctuatedWatermarks { @Override public long extractTimestamp(MyEvent element, long previousElementTimestamp) { diff --git a/docs/dev/execution_configuration.md b/docs/dev/execution_configuration.md index 8fe1b6376e701d..fc2364bdfa047d 100644 --- a/docs/dev/execution_configuration.md +++ b/docs/dev/execution_configuration.md @@ -51,11 +51,11 @@ With the closure cleaner disabled, it might happen that an anonymous user functi - `getNumberOfExecutionRetries()` / `setNumberOfExecutionRetries(int numberOfExecutionRetries)` Sets the number of times that failed tasks are re-executed. A value of zero effectively disables fault tolerance. A value of `-1` indicates that the system default value (as defined in the configuration) should be used. This is deprecated, use [restart strategies]({{ site.baseurl }}/dev/restart_strategies.html) instead. -- `getExecutionRetryDelay()` / `setExecutionRetryDelay(long executionRetryDelay)` Sets the delay in milliseconds that the system waits after a job has failed, before re-executing it. The delay starts after all tasks have been successfully been stopped on the TaskManagers, and once the delay is past, the tasks are re-started. This parameter is useful to delay re-execution in order to let certain time-out related failures surface fully (like broken connections that have not fully timed out), before attempting a re-execution and immediately failing again due to the same problem. This parameter only has an effect if the number of execution re-tries is one or more. This is deprecated, use [restart strategies]({{ site.baseurl }}/dev/restart_strategies.html) instead. +- `getExecutionRetryDelay()` / `setExecutionRetryDelay(long executionRetryDelay)` Sets the delay in milliseconds that the system waits after a job has failed, before re-executing it. The delay starts after all tasks have been successfully stopped on the TaskManagers, and once the delay is past, the tasks are re-started. This parameter is useful to delay re-execution in order to let certain time-out related failures surface fully (like broken connections that have not fully timed out), before attempting a re-execution and immediately failing again due to the same problem. This parameter only has an effect if the number of execution re-tries is one or more. This is deprecated, use [restart strategies]({{ site.baseurl }}/dev/restart_strategies.html) instead. - `getExecutionMode()` / `setExecutionMode()`. The default execution mode is PIPELINED. Sets the execution mode to execute the program. The execution mode defines whether data exchanges are performed in a batch or on a pipelined manner. -- `enableForceKryo()` / **`disableForceKryo`**. Kryo is not forced by default. Forces the GenericTypeInformation to use the Kryo serializer for POJOS even though we could analyze them as a POJO. In some cases this might be preferable. For example, when Flink's internal serializers fail to handle a POJO properly. +- `enableForceKryo()` / **`disableForceKryo`**. Kryo is not forced by default. Forces the GenericTypeInformation to use the Kryo serializer for POJOs even though we could analyze them as a POJO. In some cases this might be preferable. For example, when Flink's internal serializers fail to handle a POJO properly. - `enableForceAvro()` / **`disableForceAvro()`**. Avro is not forced by default. Forces the Flink AvroTypeInformation to use the Avro serializer instead of Kryo for serializing Avro POJOs. diff --git a/docs/dev/execution_plans.md b/docs/dev/execution_plans.md index 881c54ea3d334a..29a3641c3569c8 100644 --- a/docs/dev/execution_plans.md +++ b/docs/dev/execution_plans.md @@ -72,7 +72,7 @@ __Web Interface__ Flink offers a web interface for submitting and executing jobs. The interface is part of the JobManager's web interface for monitoring, per default running on port 8081. Job submission via this interfaces requires -that you have set `jobmanager.web.submit.enable: true` in `flink-conf.yaml`. +that you have set `web.submit.enable: true` in `flink-conf.yaml`. You may specify program arguments before the job is executed. The plan visualization enables you to show the execution plan before executing the Flink job. diff --git a/docs/dev/index.md b/docs/dev/index.md index 8b96672eeed14e..b58120471b371c 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -4,6 +4,7 @@ nav-id: dev nav-title: ' Application Development' nav-parent_id: root nav-pos: 5 +section-break: true --- - -Java 8 introduces several new language features designed for faster and clearer coding. With the most important feature, -the so-called "Lambda Expressions", Java 8 opens the door to functional programming. Lambda Expressions allow for implementing and -passing functions in a straightforward way without having to declare additional (anonymous) classes. - -The newest version of Flink supports the usage of Lambda Expressions for all operators of the Java API. -This document shows how to use Lambda Expressions and describes current limitations. For a general introduction to the -Flink API, please refer to the [Programming Guide]({{ site.baseurl }}/dev/api_concepts.html) - -* TOC -{:toc} - -### Examples - -The following example illustrates how to implement a simple, inline `map()` function that squares its input using a Lambda Expression. -The types of input `i` and output parameters of the `map()` function need not to be declared as they are inferred by the Java 8 compiler. - -~~~java -env.fromElements(1, 2, 3) -// returns the squared i -.map(i -> i*i) -.print(); -~~~ - -The next two examples show different implementations of a function that uses a `Collector` for output. -Functions, such as `flatMap()`, require an output type (in this case `String`) to be defined for the `Collector` in order to be type-safe. -If the `Collector` type can not be inferred from the surrounding context, it needs to be declared in the Lambda Expression's parameter list manually. -Otherwise the output will be treated as type `Object` which can lead to undesired behaviour. - -~~~java -DataSet input = env.fromElements(1, 2, 3); - -// collector type must be declared -input.flatMap((Integer number, Collector out) -> { - StringBuilder builder = new StringBuilder(); - for(int i = 0; i < number; i++) { - builder.append("a"); - out.collect(builder.toString()); - } -}) -// returns (on separate lines) "a", "a", "aa", "a", "aa", "aaa" -.print(); -~~~ - -~~~java -DataSet input = env.fromElements(1, 2, 3); - -// collector type must not be declared, it is inferred from the type of the dataset -DataSet manyALetters = input.flatMap((number, out) -> { - StringBuilder builder = new StringBuilder(); - for(int i = 0; i < number; i++) { - builder.append("a"); - out.collect(builder.toString()); - } -}); - -// returns (on separate lines) "a", "a", "aa", "a", "aa", "aaa" -manyALetters.print(); -~~~ - -The following code demonstrates a word count which makes extensive use of Lambda Expressions. - -~~~java -DataSet input = env.fromElements("Please count", "the words", "but not this"); - -// filter out strings that contain "not" -input.filter(line -> !line.contains("not")) -// split each line by space -.map(line -> line.split(" ")) -// emit a pair for each array element -.flatMap((String[] wordArray, Collector> out) - -> Arrays.stream(wordArray).forEach(t -> out.collect(new Tuple2<>(t, 1))) - ) -// group and sum up -.groupBy(0).sum(1) -// print -.print(); -~~~ - -### Compiler Limitations -Currently, Flink only supports jobs containing Lambda Expressions completely if they are **compiled with the Eclipse JDT compiler contained in Eclipse Luna 4.4.2 (and above)**. - -Only the Eclipse JDT compiler preserves the generic type information necessary to use the entire Lambda Expressions feature type-safely. -Other compilers such as the OpenJDK's and Oracle JDK's `javac` throw away all generic parameters related to Lambda Expressions. This means that types such as `Tuple2` or `Collector` declared as a Lambda function input or output parameter will be pruned to `Tuple2` or `Collector` in the compiled `.class` files, which is too little information for the Flink compiler. - -How to compile a Flink job that contains Lambda Expressions with the JDT compiler will be covered in the next section. - -However, it is possible to implement functions such as `map()` or `filter()` with Lambda Expressions in Java 8 compilers other than the Eclipse JDT compiler as long as the function has no `Collector`s or `Iterable`s *and* only if the function handles unparameterized types such as `Integer`, `Long`, `String`, `MyOwnClass` (types without Generics!). - -#### Compile Flink jobs with the Eclipse JDT compiler and Maven - -If you are using the Eclipse IDE, you can run and debug your Flink code within the IDE without any problems after some configuration steps. The Eclipse IDE by default compiles its Java sources with the Eclipse JDT compiler. The next section describes how to configure the Eclipse IDE. - -If you are using a different IDE such as IntelliJ IDEA or you want to package your Jar-File with Maven to run your job on a cluster, you need to modify your project's `pom.xml` file and build your program with Maven. The [quickstart]({{site.baseurl}}/quickstart/setup_quickstart.html) contains preconfigured Maven projects which can be used for new projects or as a reference. Uncomment the mentioned lines in your generated quickstart `pom.xml` file if you want to use Java 8 with Lambda Expressions. - -Alternatively, you can manually insert the following lines to your Maven `pom.xml` file. Maven will then use the Eclipse JDT compiler for compilation. - -~~~xml - - - - - maven-compiler-plugin - - 1.8 - 1.8 - jdt - - - - - org.eclipse.tycho - tycho-compiler-jdt - 0.21.0 - - - -~~~ - -If you are using Eclipse for development, the m2e plugin might complain about the inserted lines above and marks your `pom.xml` as invalid. If so, insert the following lines to your `pom.xml`. - -~~~xml - - - - - org.apache.maven.plugins - maven-compiler-plugin - [3.1,) - - testCompile - compile - - - - - - -~~~ - -#### Run and debug Flink jobs within the Eclipse IDE - -First of all, make sure you are running a current version of Eclipse IDE (4.4.2 or later). Also make sure that you have a Java 8 Runtime Environment (JRE) installed in Eclipse IDE (`Window` -> `Preferences` -> `Java` -> `Installed JREs`). - -Create/Import your Eclipse project. - -If you are using Maven, you also need to change the Java version in your `pom.xml` for the `maven-compiler-plugin`. Otherwise right click the `JRE System Library` section of your project and open the `Properties` window in order to switch to a Java 8 JRE (or above) that supports Lambda Expressions. - -The Eclipse JDT compiler needs a special compiler flag in order to store type information in `.class` files. Open the JDT configuration file at `{project directory}/.settings/org.eclipse.jdt.core.prefs` with your favorite text editor and add the following line: - -~~~ -org.eclipse.jdt.core.compiler.codegen.lambda.genericSignature=generate -~~~ - -If not already done, also modify the Java versions of the following properties to `1.8` (or above): - -~~~ -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.source=1.8 -~~~ - -After you have saved the file, perform a complete project refresh in Eclipse IDE. - -If you are using Maven, right click your Eclipse project and select `Maven` -> `Update Project...`. - -You have configured everything correctly, if the following Flink program runs without exceptions: - -~~~java -final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); -env.fromElements(1, 2, 3).map((in) -> new Tuple1(" " + in)).print(); -env.execute(); -~~~ - -{% top %} diff --git a/docs/dev/java_lambdas.md b/docs/dev/java_lambdas.md new file mode 100644 index 00000000000000..bc9704b517d0be --- /dev/null +++ b/docs/dev/java_lambdas.md @@ -0,0 +1,140 @@ +--- +title: "Java Lambda Expressions" +nav-parent_id: api-concepts +nav-pos: 20 +--- + + +Java 8 introduced several new language features designed for faster and clearer coding. With the most important feature, +the so-called "Lambda Expressions", it opened the door to functional programming. Lambda expressions allow for implementing and +passing functions in a straightforward way without having to declare additional (anonymous) classes. + +Attention Flink supports the usage of lambda expressions for all operators of the Java API, however, whenever a lambda expression uses Java generics you need to declare type information *explicitly*. + +This document shows how to use lambda expressions and describes current limitations. For a general introduction to the +Flink API, please refer to the [Programming Guide]({{ site.baseurl }}/dev/api_concepts.html) + +### Examples and Limitations + +The following example illustrates how to implement a simple, inline `map()` function that squares its input using a lambda expression. +The types of input `i` and output parameters of the `map()` function need not to be declared as they are inferred by the Java compiler. + +{% highlight java %} +env.fromElements(1, 2, 3) +// returns the squared i +.map(i -> i*i) +.print(); +{% endhighlight %} + +Flink can automatically extract the result type information from the implementation of the method signature `OUT map(IN value)` because `OUT` is not generic but `Integer`. + +Unfortunately, functions such as `flatMap()` with a signature `void flatMap(IN value, Collector out)` are compiled into `void flatMap(IN value, Collector out)` by the Java compiler. This makes it impossible for Flink to infer the type information for the output type automatically. + +Flink will most likely throw an exception similar to the following: + +{% highlight plain%} +org.apache.flink.api.common.functions.InvalidTypesException: The generic type parameters of 'Collector' are missing. + In many cases lambda methods don't provide enough information for automatic type extraction when Java generics are involved. + An easy workaround is to use an (anonymous) class instead that implements the 'org.apache.flink.api.common.functions.FlatMapFunction' interface. + Otherwise the type has to be specified explicitly using type information. +{% endhighlight %} + +In this case, the type information needs to be *specified explicitly*, otherwise the output will be treated as type `Object` which leads to unefficient serialization. + +{% highlight java %} +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.util.Collector; + +DataSet input = env.fromElements(1, 2, 3); + +// collector type must be declared +input.flatMap((Integer number, Collector out) -> { + StringBuilder builder = new StringBuilder(); + for(int i = 0; i < number; i++) { + builder.append("a"); + out.collect(builder.toString()); + } +}) +// provide type information explicitly +.returns(Types.STRING) +// prints "a", "a", "aa", "a", "aa", "aaa" +.print(); +{% endhighlight %} + +Similar problems occur when using a `map()` function with a generic return type. A method signature `Tuple2 map(Integer value)` is erasured to `Tuple2 map(Integer value)` in the example below. + +{% highlight java %} +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.tuple.Tuple2; + +env.fromElements(1, 2, 3) + .map(i -> Tuple2.of(i, i)) // no information about fields of Tuple2 + .print(); +{% endhighlight %} + +In general, those problems can be solved in multiple ways: + +{% highlight java %} +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.tuple.Tuple2; + +// use the explicit ".returns(...)" +env.fromElements(1, 2, 3) + .map(i -> Tuple2.of(i, i)) + .returns(Types.TUPLE(Types.INT, Types.INT)) + .print(); + +// use a class instead +env.fromElements(1, 2, 3) + .map(new MyTuple2Mapper()) + .print(); + +public static class MyTuple2Mapper extends MapFunction> { + @Override + public Tuple2 map(Integer i) { + return Tuple2.of(i, i); + } +} + +// use an anonymous class instead +env.fromElements(1, 2, 3) + .map(new MapFunction> { + @Override + public Tuple2 map(Integer i) { + return Tuple2.of(i, i); + } + }) + .print(); + +// or in this example use a tuple subclass instead +env.fromElements(1, 2, 3) + .map(i -> new DoubleTuple(i, i)) + .print(); + +public static class DoubleTuple extends Tuple2 { + public DoubleTuple(int f0, int f1) { + this.f0 = f0; + this.f1 = f1; + } +} +{% endhighlight %} + +{% top %} diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md index a6136d11e9dd98..a358ba2bc1e0e8 100644 --- a/docs/dev/libs/cep.md +++ b/docs/dev/libs/cep.md @@ -31,7 +31,7 @@ This page describes the API calls available in Flink CEP. We start by presenting which allows you to specify the patterns that you want to detect in your stream, before presenting how you can [detect and act upon matching event sequences](#detecting-patterns). We then present the assumptions the CEP library makes when [dealing with lateness](#handling-lateness-in-event-time) in event time and how you can -[migrate your job](#migrating-from-an-older-flink-version) from an older Flink version to Flink-1.3. +[migrate your job](#migrating-from-an-older-flink-versionpre-13) from an older Flink version to Flink-1.3. * This will be replaced by the TOC {:toc} @@ -84,7 +84,7 @@ Pattern pattern = Pattern.begin("start").where( } } ).next("middle").subtype(SubEvent.class).where( - new SimpleCondition() { + new SimpleCondition() { @Override public boolean filter(SubEvent subEvent) { return subEvent.getVolume() >= 10.0; @@ -101,27 +101,37 @@ Pattern pattern = Pattern.begin("start").where( PatternStream patternStream = CEP.pattern(input, pattern); -DataStream result = patternStream.select( - new PatternSelectFunction { +DataStream result = patternStream.process( + new PatternProcessFunction() { @Override - public Alert select(Map> pattern) throws Exception { - return createAlertFrom(pattern); + public void processMatch( + Map> pattern, + Context ctx, + Collector out) throws Exception { + out.collect(createAlertFrom(pattern)); } - } -}); + }); {% endhighlight %}
    {% highlight scala %} val input: DataStream[Event] = ... -val pattern = Pattern.begin("start").where(_.getId == 42) +val pattern = Pattern.begin[Event]("start").where(_.getId == 42) .next("middle").subtype(classOf[SubEvent]).where(_.getVolume >= 10.0) .followedBy("end").where(_.getName == "end") val patternStream = CEP.pattern(input, pattern) -val result: DataStream[Alert] = patternStream.select(createAlert(_)) +val result: DataStream[Alert] = patternStream.process( + new PatternProcessFunction[Event, Alert]() { + override def processMatch( + `match`: util.Map[String, util.List[Event]], + ctx: PatternProcessFunction.Context, + out: Collector[Alert]): Unit = { + out.collect(createAlertFrom(pattern)) + } + }) {% endhighlight %}
    @@ -131,7 +141,7 @@ val result: DataStream[Alert] = patternStream.select(createAlert(_)) The pattern API allows you to define complex pattern sequences that you want to extract from your input stream. Each complex pattern sequence consists of multiple simple patterns, i.e. patterns looking for individual events with the same properties. From now on, we will call these simple patterns **patterns**, and the final complex pattern sequence we are searching for in the stream, the **pattern sequence**. You can see a pattern sequence as a graph of such patterns, where transitions from one pattern to the next occur based on user-specified -*conditions*, e.g. `event.getName().equals("start")`. A **match** is a sequence of input events which visits all +*conditions*, e.g. `event.getName().equals("end")`. A **match** is a sequence of input events which visits all patterns of the complex pattern graph, through a sequence of valid pattern transitions. {% warn Attention %} Each pattern must have a unique name, which you use later to identify the matched events. @@ -250,21 +260,10 @@ For a pattern named `start`, the following are valid quantifiers: #### Conditions -At every pattern, and to go from one pattern to the next, you can specify additional **conditions**. -You can relate these conditions to: - - 1. A [property of the incoming event](#conditions-on-properties), e.g. its value should be larger than 5, - or larger than the average value of the previously accepted events. - - 2. The [contiguity of the matching events](#conditions-on-contiguity), e.g. detect pattern `a,b,c` without - non-matching events between any matching ones. - -The latter refers to "looping" patterns, *i.e.* patterns that can accept more than one event, e.g. the `b+` in `a b+ c`, -which searches for one or more `b`'s. - -##### Conditions on Properties - -You can specify conditions on the event properties via the `pattern.where()`, `pattern.or()` or the `pattern.until()` method. These can be either `IterativeCondition`s or `SimpleCondition`s. +For every pattern you can specify a condition that an incoming event has to meet in order to be "accepted" into the pattern e.g. its value should be larger than 5, +or larger than the average value of the previously accepted events. +You can specify conditions on the event properties via the `pattern.where()`, `pattern.or()` or `pattern.until()` methods. +These can be either `IterativeCondition`s or `SimpleCondition`s. **Iterative Conditions:** This is the most general type of condition. This is how you can specify a condition that accepts subsequent events based on properties of the previously accepted events or a statistic over a subset of them. @@ -275,39 +274,45 @@ with "foo", and if the sum of the prices of the previously accepted events for t
    {% highlight java %} -middle.oneOrMore().where(new IterativeCondition() { - @Override - public boolean filter(SubEvent value, Context ctx) throws Exception { - if (!value.getName().startsWith("foo")) { - return false; - } - - double sum = value.getPrice(); - for (Event event : ctx.getEventsForPattern("middle")) { - sum += event.getPrice(); +middle.oneOrMore() + .subtype(SubEvent.class) + .where(new IterativeCondition() { + @Override + public boolean filter(SubEvent value, Context ctx) throws Exception { + if (!value.getName().startsWith("foo")) { + return false; + } + + double sum = value.getPrice(); + for (Event event : ctx.getEventsForPattern("middle")) { + sum += event.getPrice(); + } + return Double.compare(sum, 5.0) < 0; } - return Double.compare(sum, 5.0) < 0; - } -}); + }); {% endhighlight %}
    {% highlight scala %} -middle.oneOrMore().where( - (value, ctx) => { - lazy val sum = ctx.getEventsForPattern("middle").asScala.map(_.getPrice).sum - value.getName.startsWith("foo") && sum + value.getPrice < 5.0 - } -) +middle.oneOrMore() + .subtype(classOf[SubEvent]) + .where( + (value, ctx) => { + lazy val sum = ctx.getEventsForPattern("middle").map(_.getPrice).sum + value.getName.startsWith("foo") && sum + value.getPrice < 5.0 + } + ) {% endhighlight %}
    -{% warn Attention %} The call to `context.getEventsForPattern(...)` finds all the +{% warn Attention %} The call to `ctx.getEventsForPattern(...)` finds all the previously accepted events for a given potential match. The cost of this operation can vary, so when implementing your condition, try to minimize its use. +Described context gives one access to event time characteristics as well. For more info see [Time context](#time-context). + **Simple Conditions:** This type of condition extends the aforementioned `IterativeCondition` class and decides whether to accept an event or not, based *only* on properties of the event itself. @@ -392,36 +397,6 @@ To better understand it, have a look at the following example. Given As you can see `{a1 a2 a3}` or `{a2 a3}` are not returned due to the stop condition. -##### Conditions on Contiguity - -FlinkCEP supports the following forms of contiguity between events: - - 1. **Strict Contiguity**: Expects all matching events to appear strictly one after the other, without any non-matching events in-between. - - 2. **Relaxed Contiguity**: Ignores non-matching events appearing in-between the matching ones. - - 3. **Non-Deterministic Relaxed Contiguity**: Further relaxes contiguity, allowing additional matches - that ignore some matching events. - -To illustrate the above with an example, a pattern sequence `"a+ b"` (one or more `"a"`'s followed by a `"b"`) with -input `"a1", "c", "a2", "b"` will have the following results: - - 1. **Strict Contiguity**: `{a2 b}` -- the `"c"` after `"a1"` causes `"a1"` to be discarded. - - 2. **Relaxed Contiguity**: `{a1 b}` and `{a1 a2 b}` -- `c` is ignored. - - 3. **Non-Deterministic Relaxed Contiguity**: `{a1 b}`, `{a2 b}`, and `{a1 a2 b}`. - -For looping patterns (e.g. `oneOrMore()` and `times()`) the default is *relaxed contiguity*. If you want -strict contiguity, you have to explicitly specify it by using the `consecutive()` call, and if you want -*non-deterministic relaxed contiguity* you can use the `allowCombinations()` call. - -{% warn Attention %} -In this section we are talking about contiguity *within* a single looping pattern, and the -`consecutive()` and `allowCombinations()` calls need to be understood in that context. Later when looking at -[Combining Patterns](#combining-patterns) we'll discuss other calls, such as `next()` and `followedBy()`, -that are used to specify contiguity conditions *between* patterns. -
    @@ -561,74 +536,6 @@ pattern.oneOrMore().greedy(); {% endhighlight %} - - - - - - - -
    consecutive() -

    Works in conjunction with oneOrMore() and times() and imposes strict contiguity between the matching - events, i.e. any non-matching element breaks the match (as in next()).

    -

    If not applied a relaxed contiguity (as in followedBy()) is used.

    - -

    E.g. a pattern like:

    -{% highlight java %} -Pattern.begin("start").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("c"); - } -}) -.followedBy("middle").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("a"); - } -}).oneOrMore().consecutive() -.followedBy("end1").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("b"); - } -}); -{% endhighlight %} -

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    - -

    with consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}

    -

    without consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    -
    allowCombinations() -

    Works in conjunction with oneOrMore() and times() and imposes non-deterministic relaxed contiguity - between the matching events (as in followedByAny()).

    -

    If not applied a relaxed contiguity (as in followedBy()) is used.

    - -

    E.g. a pattern like:

    -{% highlight java %} -Pattern.begin("start").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("c"); - } -}) -.followedBy("middle").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("a"); - } -}).oneOrMore().allowCombinations() -.followedBy("end1").where(new SimpleCondition() { - @Override - public boolean filter(Event value) throws Exception { - return value.getName().equals("b"); - } -}); -{% endhighlight %} -

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    - -

    with combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A3 B}, {C A1 A4 B}, {C A1 A2 A3 B}, {C A1 A2 A4 B}, {C A1 A3 A4 B}, {C A1 A2 A3 A4 B}

    -

    without combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    -
    @@ -753,52 +660,9 @@ pattern.oneOrMore().greedy() {% endhighlight %} - - consecutive() - -

    Works in conjunction with oneOrMore() and times() and imposes strict contiguity between the matching - events, i.e. any non-matching element breaks the match (as in next()).

    -

    If not applied a relaxed contiguity (as in followedBy()) is used.

    - -

    E.g. a pattern like:

    -{% highlight scala %} -Pattern.begin("start").where(_.getName().equals("c")) - .followedBy("middle").where(_.getName().equals("a")) - .oneOrMore().consecutive() - .followedBy("end1").where(_.getName().equals("b")) -{% endhighlight %} - -

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    - -

    with consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}

    -

    without consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    - - - - allowCombinations() - -

    Works in conjunction with oneOrMore() and times() and imposes non-deterministic relaxed contiguity - between the matching events (as in followedByAny()).

    -

    If not applied a relaxed contiguity (as in followedBy()) is used.

    - -

    E.g. a pattern like:

    -{% highlight scala %} -Pattern.begin("start").where(_.getName().equals("c")) - .followedBy("middle").where(_.getName().equals("a")) - .oneOrMore().allowCombinations() - .followedBy("end1").where(_.getName().equals("b")) -{% endhighlight %} - -

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    - -

    with combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A3 B}, {C A1 A4 B}, {C A1 A2 A3 B}, {C A1 A2 A4 B}, {C A1 A3 A4 B}, {C A1 A2 A3 A4 B}

    -

    without combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    - -
    -
    ### Combining Patterns @@ -823,9 +687,16 @@ val start : Pattern[Event, _] = Pattern.begin("start")
    Next, you can append more patterns to your pattern sequence by specifying the desired *contiguity conditions* between -them. In the [previous section](#conditions-on-contiguity) we described the different contiguity modes supported by -Flink, namely *strict*, *relaxed*, and *non-deterministic relaxed*, and how to apply them in looping patterns. To apply -them between consecutive patterns, you can use: +them. FlinkCEP supports the following forms of contiguity between events: + + 1. **Strict Contiguity**: Expects all matching events to appear strictly one after the other, without any non-matching events in-between. + + 2. **Relaxed Contiguity**: Ignores non-matching events appearing in-between the matching ones. + + 3. **Non-Deterministic Relaxed Contiguity**: Further relaxes contiguity, allowing additional matches + that ignore some matching events. + +To apply them between consecutive patterns, you can use: 1. `next()`, for *strict*, 2. `followedBy()`, for *relaxed*, and @@ -834,7 +705,7 @@ them between consecutive patterns, you can use: or 1. `notNext()`, if you do not want an event type to directly follow another -2. `notFollowedBy()`, if you do not want an event type to be anywhere between two other event types +2. `notFollowedBy()`, if you do not want an event type to be anywhere between two other event types. {% warn Attention %} A pattern sequence cannot end in `notFollowedBy()`. @@ -886,14 +757,14 @@ val relaxedNot: Pattern[Event, _] = start.notFollowedBy("not").where(...) Relaxed contiguity means that only the first succeeding matching event will be matched, while with non-deterministic relaxed contiguity, multiple matches will be emitted for the same beginning. As an example, -a pattern `a b`, given the event sequence `"a", "c", "b1", "b2"`, will give the following results: +a pattern `"a b"`, given the event sequence `"a", "c", "b1", "b2"`, will give the following results: -1. Strict Contiguity between `a` and `b`: `{}` (no match), the `"c"` after `"a"` causes `"a"` to be discarded. +1. Strict Contiguity between `"a"` and `"b"`: `{}` (no match), the `"c"` after `"a"` causes `"a"` to be discarded. -2. Relaxed Contiguity between `a` and `b`: `{a b1}`, as relaxed continuity is viewed as "skip non-matching events +2. Relaxed Contiguity between `"a"` and `"b"`: `{a b1}`, as relaxed continuity is viewed as "skip non-matching events till the next matching one". -3. Non-Deterministic Relaxed Contiguity between `a` and `b`: `{a b1}`, `{a b2}`, as this is the most general form. +3. Non-Deterministic Relaxed Contiguity between `"a"` and `"b"`: `{a b1}`, `{a b2}`, as this is the most general form. It's also possible to define a temporal constraint for the pattern to be valid. For example, you can define that a pattern should occur within 10 seconds via the `pattern.within()` method. @@ -915,6 +786,164 @@ next.within(Time.seconds(10))
    +#### Contiguity within looping patterns + +You can apply the same contiguity condition as discussed in the previous [section](#combining-patterns) within a looping pattern. +The contiguity will be applied between elements accepted into such a pattern. +To illustrate the above with an example, a pattern sequence `"a b+ c"` (`"a"` followed by any(non-deterministic relaxed) sequence of one or more `"b"`'s followed by a `"c"`) with +input `"a", "b1", "d1", "b2", "d2", "b3" "c"` will have the following results: + + 1. **Strict Contiguity**: `{a b3 c}` -- the `"d1"` after `"b1"` causes `"b1"` to be discarded, the same happens for `"b2"` because of `"d2"`. + + 2. **Relaxed Contiguity**: `{a b1 c}`, `{a b1 b2 c}`, `{a b1 b2 b3 c}`, `{a b2 c}`, `{a b2 b3 c}`, `{a b3 c}` - `"d"`'s are ignored. + + 3. **Non-Deterministic Relaxed Contiguity**: `{a b1 c}`, `{a b1 b2 c}`, `{a b1 b3 c}`, `{a b1 b2 b3 c}`, `{a b2 c}`, `{a b2 b3 c}`, `{a b3 c}` - + notice the `{a b1 b3 c}`, which is the result of relaxing contiguity between `"b"`'s. + +For looping patterns (e.g. `oneOrMore()` and `times()`) the default is *relaxed contiguity*. If you want +strict contiguity, you have to explicitly specify it by using the `consecutive()` call, and if you want +*non-deterministic relaxed contiguity* you can use the `allowCombinations()` call. + +
    +
    + + + + + + + + + + + + + + + + + +
    Pattern OperationDescription
    consecutive() +

    Works in conjunction with oneOrMore() and times() and imposes strict contiguity between the matching + events, i.e. any non-matching element breaks the match (as in next()).

    +

    If not applied a relaxed contiguity (as in followedBy()) is used.

    + +

    E.g. a pattern like:

    +{% highlight java %} +Pattern.begin("start").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("c"); + } +}) +.followedBy("middle").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("a"); + } +}).oneOrMore().consecutive() +.followedBy("end1").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("b"); + } +}); +{% endhighlight %} +

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    + +

    with consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}

    +

    without consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    +
    allowCombinations() +

    Works in conjunction with oneOrMore() and times() and imposes non-deterministic relaxed contiguity + between the matching events (as in followedByAny()).

    +

    If not applied a relaxed contiguity (as in followedBy()) is used.

    + +

    E.g. a pattern like:

    +{% highlight java %} +Pattern.begin("start").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("c"); + } +}) +.followedBy("middle").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("a"); + } +}).oneOrMore().allowCombinations() +.followedBy("end1").where(new SimpleCondition() { + @Override + public boolean filter(Event value) throws Exception { + return value.getName().equals("b"); + } +}); +{% endhighlight %} +

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    + +

    with combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A3 B}, {C A1 A4 B}, {C A1 A2 A3 B}, {C A1 A2 A4 B}, {C A1 A3 A4 B}, {C A1 A2 A3 A4 B}

    +

    without combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    +
    +
    + +
    + + + + + + + + + + + + + + + + + +
    Pattern OperationDescription
    consecutive() +

    Works in conjunction with oneOrMore() and times() and imposes strict contiguity between the matching + events, i.e. any non-matching element breaks the match (as in next()).

    +

    If not applied a relaxed contiguity (as in followedBy()) is used.

    + +

    E.g. a pattern like:

    +{% highlight scala %} +Pattern.begin("start").where(_.getName().equals("c")) + .followedBy("middle").where(_.getName().equals("a")) + .oneOrMore().consecutive() + .followedBy("end1").where(_.getName().equals("b")) +{% endhighlight %} + +

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    + +

    with consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}

    +

    without consecutive applied: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    +
    allowCombinations() +

    Works in conjunction with oneOrMore() and times() and imposes non-deterministic relaxed contiguity + between the matching events (as in followedByAny()).

    +

    If not applied a relaxed contiguity (as in followedBy()) is used.

    + +

    E.g. a pattern like:

    +{% highlight scala %} +Pattern.begin("start").where(_.getName().equals("c")) + .followedBy("middle").where(_.getName().equals("a")) + .oneOrMore().allowCombinations() + .followedBy("end1").where(_.getName().equals("b")) +{% endhighlight %} + +

    Will generate the following matches for an input sequence: C D A1 A2 A3 D A4 B

    + +

    with combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A3 B}, {C A1 A4 B}, {C A1 A2 A3 B}, {C A1 A2 A4 B}, {C A1 A3 A4 B}, {C A1 A2 A3 A4 B}

    +

    without combinations enabled: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}, {C A1 A2 A3 A4 B}

    +
    +
    +
    + +### Groups of patterns + It's also possible to define a pattern sequence as the condition for `begin`, `followedBy`, `followedByAny` and `next`. The pattern sequence will be considered as the matching condition logically and a `GroupPattern` will be returned and it is possible to apply `oneOrMore()`, `times(#ofTimes)`, `times(#fromTimes, #toTimes)`, `optional()`, @@ -950,22 +979,22 @@ Pattern nonDetermin = start.followedByAny( {% highlight scala %} val start: Pattern[Event, _] = Pattern.begin( - Pattern.begin[Event, _]("start").where(...).followedBy("start_middle").where(...) + Pattern.begin[Event]("start").where(...).followedBy("start_middle").where(...) ) // strict contiguity val strict: Pattern[Event, _] = start.next( - Pattern.begin[Event, _]("next_start").where(...).followedBy("next_middle").where(...) + Pattern.begin[Event]("next_start").where(...).followedBy("next_middle").where(...) ).times(3) // relaxed contiguity val relaxed: Pattern[Event, _] = start.followedBy( - Pattern.begin[Event, _]("followedby_start").where(...).followedBy("followedby_middle").where(...) + Pattern.begin[Event]("followedby_start").where(...).followedBy("followedby_middle").where(...) ).oneOrMore() // non-deterministic relaxed contiguity val nonDetermin: Pattern[Event, _] = start.followedByAny( - Pattern.begin[Event, _]("followedbyany_start").where(...).followedBy("followedbyany_middle").where(...) + Pattern.begin[Event]("followedbyany_start").where(...).followedBy("followedbyany_middle").where(...) ).optional() {% endhighlight %} @@ -1117,11 +1146,22 @@ pattern.within(Time.seconds(10)); - begin() + begin(#name)

    Defines a starting pattern:

    {% highlight scala %} val start = Pattern.begin[Event]("start") +{% endhighlight %} + + + + begin(#pattern_sequence) + +

    Defines a starting pattern:

    +{% highlight scala %} +val start = Pattern.begin( + Pattern.begin[Event]("start").where(...).followedBy("middle").where(...) +) {% endhighlight %} @@ -1237,13 +1277,13 @@ pattern.within(Time.seconds(10)) For a given pattern, the same event may be assigned to multiple successful matches. To control to how many matches an event will be assigned, you need to specify the skip strategy called `AfterMatchSkipStrategy`. There are four types of skip strategies, listed as follows: * *NO_SKIP*: Every possible match will be emitted. -* *SKIP_PAST_LAST_EVENT*: Discards every partial match that contains event of the match. -* *SKIP_TO_FIRST*: Discards every partial match that contains event of the match preceding the first of *PatternName*. -* *SKIP_TO_LAST*: Discards every partial match that contains event of the match preceding the last of *PatternName*. +* *SKIP_PAST_LAST_EVENT*: Discards every partial match that started after the match started but before it ended. +* *SKIP_TO_FIRST*: Discards every partial match that started after the match started but before the first event of *PatternName* occurred. +* *SKIP_TO_LAST*: Discards every partial match that started after the match started but before the last event of *PatternName* occurred. Notice that when using *SKIP_TO_FIRST* and *SKIP_TO_LAST* skip strategy, a valid *PatternName* should also be specified. -For example, for a given pattern `a b{2}` and a data stream `ab1, ab2, ab3, ab4, ab5, ab6`, the differences between these four skip strategies are as follows: +For example, for a given pattern `b+ c` and a data stream `b1 b2 b3 c`, the differences between these four skip strategies are as follows: @@ -1254,38 +1294,101 @@ For example, for a given pattern `a b{2}` and a data stream `ab1, ab2, ab3, ab4, + + + + + - + - + - + + + +
    NO_SKIP - ab1 ab2 ab3
    - ab2 ab3 ab4
    - ab3 ab4 ab5
    - ab4 ab5 ab6
    + b1 b2 b3 c
    + b2 b3 c
    + b3 c
    +     		After found matching b1 b2 b3 c, the match process will not discard any result.
    SKIP_TO_NEXT + b1 b2 b3 c
    + b2 b3 c
    + b3 c
    		After found matching ab1 ab2 ab3, the match process will not discard any result.After found matching b1 b2 b3 c, the match process will not discard any result, because no other match could start at b1.
    SKIP_PAST_LAST_EVENT - ab1 ab2 ab3
    - ab4 ab5 ab6
    + b1 b2 b3 c
    		After found matching ab1 ab2 ab3, the match process will discard all started partial matches.After found matching b1 b2 b3 c, the match process will discard all started partial matches.
    SKIP_TO_FIRST[b] - ab1 ab2 ab3
    - ab2 ab3 ab4
    - ab3 ab4 ab5
    - ab4 ab5 ab6
    + b1 b2 b3 c
    + b2 b3 c
    + b3 c
    		After found matching ab1 ab2 ab3, the match process will discard all partial matches containing ab1, which is the only event that comes before the first b.After found matching b1 b2 b3 c, the match process will try to discard all partial matches started before b1, but there are no such matches. Therefore nothing will be discarded.
    SKIP_TO_LAST[b] - ab1 ab2 ab3
    - ab3 ab4 ab5
    + b1 b2 b3 c
    + b3 c
    +     		After found matching b1 b2 b3 c, the match process will try to discard all partial matches started before b3. There is one such match b2 b3 c
    + +Have a look also at another example to better see the difference between NO_SKIP and SKIP_TO_FIRST: +Pattern: `(a | b | c) (b | c) c+.greedy d` and sequence: `a b c1 c2 c3 d` Then the results will be: + + + + + + + + + + + + + + + + + + +
    Skip StrategyResult Description
    NO_SKIP + a b c1 c2 c3 d
    + b c1 c2 c3 d
    + c1 c2 c3 d
    +     		After found matching a b c1 c2 c3 d, the match process will not discard any result.
    SKIP_TO_FIRST[c*] + a b c1 c2 c3 d
    + c1 c2 c3 d
    +     		After found matching a b c1 c2 c3 d, the match process will discard all partial matches started before c1. There is one such match b c1 c2 c3 d.
    + +To better understand the difference between NO_SKIP and SKIP_TO_NEXT take a look at following example: +Pattern: `a b+` and sequence: `a b1 b2 b3` Then the results will be: + + + + + + + + + + + + + + + + - +
    Skip StrategyResult Description
    NO_SKIP + a b1
    + a b1 b2
    + a b1 b2 b3
    +     		After found matching a b1, the match process will not discard any result.
    SKIP_TO_NEXT + a b1
    		After found matching ab1 ab2 ab3, the match process will discard all partial matches containing ab1 and ab2, which are events that comes before the last b.After found matching a b1, the match process will discard all partial matches started at a. This means neither a b1 b2 nor a b1 b2 b3 could be generated.
    @@ -1299,6 +1402,10 @@ To specify which skip strategy to use, just create an `AfterMatchSkipStrategy` b AfterMatchSkipStrategy.noSkip() Create a NO_SKIP skip strategy + + AfterMatchSkipStrategy.skipToNext() + Create a SKIP_TO_NEXT skip strategy + AfterMatchSkipStrategy.skipPastLastEvent() Create a SKIP_PAST_LAST_EVENT skip strategy @@ -1330,6 +1437,23 @@ Pattern.begin("patternName", skipStrategy)
    +{% warn Attention %} For SKIP_TO_FIRST/LAST there are two options how to handle cases when there are no elements mapped to +the specified variable. By default a NO_SKIP strategy will be used in this case. The other option is to throw exception in such situation. +One can enable this option by: + +
    +
    +{% highlight java %} +AfterMatchSkipStrategy.skipToFirst(patternName).throwExceptionOnMiss() +{% endhighlight %} +
    +
    +{% highlight scala %} +AfterMatchSkipStrategy.skipToFirst(patternName).throwExceptionOnMiss() +{% endhighlight %} +
    +
    + ## Detecting Patterns After specifying the pattern sequence you are looking for, it is time to apply it to your input stream to detect @@ -1364,140 +1488,97 @@ The input stream can be *keyed* or *non-keyed* depending on your use-case. ### Selecting from Patterns -Once you have obtained a `PatternStream` you can select from detected event sequences via the `select` or `flatSelect` methods. +Once you have obtained a `PatternStream` you can apply transformation to detected event sequences. The suggested way of doing that +is by `PatternProcessFunction`. -
    -
    -The `select()` method requires a `PatternSelectFunction` implementation. -A `PatternSelectFunction` has a `select` method which is called for each matching event sequence. +A `PatternProcessFunction` has a `processMatch` method which is called for each matching event sequence. It receives a match in the form of `Map>` where the key is the name of each pattern in your pattern sequence and the value is a list of all accepted events for that pattern (`IN` is the type of your input elements). The events for a given pattern are ordered by timestamp. The reason for returning a list of accepted events for each -pattern is that when using looping patterns (e.g. `oneToMany()` and `times()`), more than one event may be accepted for a given pattern. The selection function returns exactly one result. +pattern is that when using looping patterns (e.g. `oneToMany()` and `times()`), more than one event may be accepted for a given pattern. {% highlight java %} -class MyPatternSelectFunction implements PatternSelectFunction { +class MyPatternProcessFunction extends PatternProcessFunction { @Override - public OUT select(Map> pattern) { - IN startEvent = pattern.get("start").get(0); - IN endEvent = pattern.get("end").get(0); - return new OUT(startEvent, endEvent); + public void processMatch(Map> match, Context ctx, Collector out) throws Exception; + IN startEvent = match.get("start").get(0); + IN endEvent = match.get("end").get(0); + out.collect(OUT(startEvent, endEvent)); } } {% endhighlight %} -A `PatternFlatSelectFunction` is similar to the `PatternSelectFunction`, with the only distinction that it can return an -arbitrary number of results. To do this, the `select` method has an additional `Collector` parameter which is -used to forward your output elements downstream. +The `PatternProcessFunction` gives access to a `Context` object. Thanks to it, one can access time related +characteristics such as `currentProcessingTime` or `timestamp` of current match (which is the timestamp of the last element assigned to the match). +For more info see [Time context](#time-context). +Through this context one can also emit results to a [side-output]({{ site.baseurl }}/dev/stream/side_output.html). -{% highlight java %} -class MyPatternFlatSelectFunction implements PatternFlatSelectFunction { - @Override - public void flatSelect(Map> pattern, Collector collector) { - IN startEvent = pattern.get("start").get(0); - IN endEvent = pattern.get("end").get(0); - for (int i = 0; i < startEvent.getValue(); i++ ) { - collector.collect(new OUT(startEvent, endEvent)); - } - } -} -{% endhighlight %} -
    - -
    -The `select()` method takes a selection function as argument, which is called for each matching event sequence. -It receives a match in the form of `Map[String, Iterable[IN]]` where the key is the name of each pattern in your pattern -sequence and the value is an Iterable over all accepted events for that pattern (`IN` is the type of your input elements). - -The events for a given pattern are ordered by timestamp. The reason for returning an iterable of accepted events for each pattern is that when using looping patterns (e.g. `oneToMany()` and `times()`), more than one event may be accepted for a given pattern. The selection function returns exactly one result per call. +#### Handling Timed Out Partial Patterns -{% highlight scala %} -def selectFn(pattern : Map[String, Iterable[IN]]): OUT = { - val startEvent = pattern.get("start").get.next - val endEvent = pattern.get("end").get.next - OUT(startEvent, endEvent) -} -{% endhighlight %} +Whenever a pattern has a window length attached via the `within` keyword, it is possible that partial event sequences +are discarded because they exceed the window length. To act upon a timed out partial match one can use `TimedOutPartialMatchHandler` interface. +The interface is supposed to be used in a mixin style. This mean you can additionally implement this interface with your `PatternProcessFunction`. +The `TimedOutPartialMatchHandler` provides the additional `processTimedOutMatch` method which will be called for every timed out partial match. -The `flatSelect` method is similar to the `select` method. Their only difference is that the function passed to the -`flatSelect` method can return an arbitrary number of results per call. In order to do this, the function for -`flatSelect` has an additional `Collector` parameter which is used to forward your output elements downstream. +{% highlight java %} +class MyPatternProcessFunction extends PatternProcessFunction implements TimedOutPartialMatchHandler { + @Override + public void processMatch(Map> match, Context ctx, Collector out) throws Exception; + ... + } -{% highlight scala %} -def flatSelectFn(pattern : Map[String, Iterable[IN]], collector : Collector[OUT]) = { - val startEvent = pattern.get("start").get.next - val endEvent = pattern.get("end").get.next - for (i <- 0 to startEvent.getValue) { - collector.collect(OUT(startEvent, endEvent)) + @Override + public void processTimedOutMatch(Map> match, Context ctx) throws Exception; + IN startEvent = match.get("start").get(0); + ctx.output(outputTag, T(startEvent)); } } {% endhighlight %} -
    -
    -### Handling Timed Out Partial Patterns +Note The `processTimedOutMatch` does not give one access to the main output. You can still emit results +through [side-outputs]({{ site.baseurl }}/dev/stream/side_output.html) though, through the `Context` object. -Whenever a pattern has a window length attached via the `within` keyword, it is possible that partial event sequences -are discarded because they exceed the window length. To react to these timed out partial matches the `select` -and `flatSelect` API calls allow you to specify a timeout handler. This timeout handler is called for each timed out -partial event sequence. The timeout handler receives all the events that have been matched so far by the pattern, and -the timestamp when the timeout was detected. -To treat partial patterns, the `select` and `flatSelect` API calls offer an overloaded version which takes as -parameters +#### Convenience API - * `PatternTimeoutFunction`/`PatternFlatTimeoutFunction` - * [OutputTag]({{ site.baseurl }}/dev/stream/side_output.html) for the side output in which the timed out matches will be returned - * and the known `PatternSelectFunction`/`PatternFlatSelectFunction`. +The aforementioned `PatternProcessFunction` was introduced in Flink 1.8 and since then it is the recommended way to interact with matches. +One can still use the old style API like `select`/`flatSelect`, which internally will be translated into a `PatternProcessFunction`.
    -~~~java +{% highlight java %} PatternStream patternStream = CEP.pattern(input, pattern); OutputTag outputTag = new OutputTag("side-output"){}; -SingleOutputStreamOperator result = patternStream.select( - new PatternTimeoutFunction() {...}, - outputTag, - new PatternSelectFunction() {...} -); - -DataStream timeoutResult = result.getSideOutput(outputTag); - SingleOutputStreamOperator flatResult = patternStream.flatSelect( - new PatternFlatTimeoutFunction() {...}, outputTag, - new PatternFlatSelectFunction() {...} + new PatternFlatTimeoutFunction() { + public void timeout( + Map> pattern, + long timeoutTimestamp, + Collector out) throws Exception { + out.collect(new TimeoutEvent()); + } + }, + new PatternFlatSelectFunction() { + public void flatSelect(Map> pattern, Collector out) throws Exception { + out.collect(new ComplexEvent()); + } + } ); DataStream timeoutFlatResult = flatResult.getSideOutput(outputTag); -~~~ +{% endhighlight %}
    -~~~scala -val patternStream: PatternStream[Event] = CEP.pattern(input, pattern) - -val outputTag = OutputTag[String]("side-output") - -val result: SingleOutputStreamOperator[ComplexEvent] = patternStream.select(outputTag){ - (pattern: Map[String, Iterable[Event]], timestamp: Long) => TimeoutEvent() -} { - pattern: Map[String, Iterable[Event]] => ComplexEvent() -} - -val timeoutResult: DataStream = result.getSideOutput(outputTag) -~~~ - -The `flatSelect` API call offers the same overloaded version which takes as the first parameter a timeout function and as second parameter a selection function. -In contrast to the `select` functions, the `flatSelect` functions are called with a `Collector`. You can use the collector to emit an arbitrary number of events. +{% highlight scala %} -~~~scala val patternStream: PatternStream[Event] = CEP.pattern(input, pattern) val outputTag = OutputTag[String]("side-output") @@ -1511,12 +1592,14 @@ val result: SingleOutputStreamOperator[ComplexEvent] = patternStream.flatSelect( } val timeoutResult: DataStream = result.getSideOutput(outputTag) -~~~ +{% endhighlight %}
    -## Handling Lateness in Event Time +## Time in CEP library + +### Handling Lateness in Event Time In `CEP` the order in which elements are processed matters. To guarantee that elements are processed in the correct order when working in event time, an incoming element is initially put in a buffer where elements are *sorted in ascending order based on their timestamp*, and when a watermark arrives, all the elements in this buffer with timestamps smaller than that of the watermark are processed. This implies that elements between watermarks are processed in event-time order. @@ -1524,7 +1607,81 @@ In `CEP` the order in which elements are processed matters. To guarantee that el To guarantee that elements across watermarks are processed in event-time order, Flink's CEP library assumes *correctness of the watermark*, and considers as *late* elements whose timestamp is smaller than that of the last -seen watermark. Late elements are not further processed. +seen watermark. Late elements are not further processed. Also, you can specify a sideOutput tag to collect the late elements come after the last seen watermark, you can use it like this. + +
    +
    + +{% highlight java %} +PatternStream patternStream = CEP.pattern(input, pattern); + +OutputTag lateDataOutputTag = new OutputTag("late-data"){}; + +SingleOutputStreamOperator result = patternStream + .sideOutputLateData(lateDataOutputTag) + .select( + new PatternSelectFunction() {...} + ); + +DataStream lateData = result.getSideOutput(lateDataOutputTag); + +{% endhighlight %} + +
    + +
    + +{% highlight scala %} + +val patternStream: PatternStream[Event] = CEP.pattern(input, pattern) + +val lateDataOutputTag = OutputTag[String]("late-data") + +val result: SingleOutputStreamOperator[ComplexEvent] = patternStream + .sideOutputLateData(lateDataOutputTag) + .select{ + pattern: Map[String, Iterable[ComplexEvent]] => ComplexEvent() + } + +val lateData: DataStream = result.getSideOutput(lateDataOutputTag) + +{% endhighlight %} + +
    +
    + +### Time context + +In [PatternProcessFunction](#selecting-from-patterns) as well as in [IterativeCondition](#conditions) user has access to a context +that implements `TimeContext` as follows: + +{% highlight java %} +/** + * Enables access to time related characteristics such as current processing time or timestamp of + * currently processed element. Used in {@link PatternProcessFunction} and + * {@link org.apache.flink.cep.pattern.conditions.IterativeCondition} + */ +@PublicEvolving +public interface TimeContext { + + /** + * Timestamp of the element currently being processed. + * + *

    In case of {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime} this + * will be set to the time when event entered the cep operator. + */ + long timestamp(); + + /** Returns the current processing time. */ + long currentProcessingTime(); +} +{% endhighlight %} + +This context gives user access to time characteristics of processed events (incoming records in case of `IterativeCondition` and matches in case of `PatternProcessFunction`). +Call to `TimeContext#currentProcessingTime` always gives you the value of current processing time and this call should be preferred to e.g. calling `System.currentTimeMillis()`. + +In case of `TimeContext#timestamp()` the returned value is equal to assigned timestamp in case of `EventTime`. In `ProcessingTime` this will equal to the point of time when said event entered +cep operator (or when the match was generated in case of `PatternProcessFunction`). This means that the value will be consistent across multiple calls to that method. ## Examples @@ -1580,7 +1737,7 @@ val input : DataStream[Event] = ... val partitionedInput = input.keyBy(event => event.getId) -val pattern = Pattern.begin("start") +val pattern = Pattern.begin[Event]("start") .next("middle").where(_.getName == "error") .followedBy("end").where(_.getName == "critical") .within(Time.seconds(10)) diff --git a/docs/dev/libs/gelly/index.md b/docs/dev/libs/gelly/index.md index 7ae796824a0a33..2a08ac256d8a42 100644 --- a/docs/dev/libs/gelly/index.md +++ b/docs/dev/libs/gelly/index.md @@ -76,35 +76,35 @@ in the **opt** directory (for versions older than Flink 1.2 these can be manuall [Maven Central](http://search.maven.org/#search|ga|1|flink%20gelly)). To run the Gelly examples the **flink-gelly** (for Java) or **flink-gelly-scala** (for Scala) jar must be copied to Flink's **lib** directory. -~~~bash +{% highlight bash %} cp opt/flink-gelly_*.jar lib/ cp opt/flink-gelly-scala_*.jar lib/ -~~~ +{% endhighlight %} Gelly's examples jar includes drivers for each of the library methods and is provided in the **examples** directory. After configuring and starting the cluster, list the available algorithm classes: -~~~bash +{% highlight bash %} ./bin/start-cluster.sh ./bin/flink run examples/gelly/flink-gelly-examples_*.jar -~~~ +{% endhighlight %} The Gelly drivers can generate graph data or read the edge list from a CSV file (each node in a cluster must have access to the input file). The algorithm description, available inputs and outputs, and configuration are displayed when an algorithm is selected. Print usage for [JaccardIndex](./library_methods.html#jaccard-index): -~~~bash +{% highlight bash %} ./bin/flink run examples/gelly/flink-gelly-examples_*.jar --algorithm JaccardIndex -~~~ +{% endhighlight %} Display [graph metrics](./library_methods.html#metric) for a million vertex graph: -~~~bash +{% highlight bash %} ./bin/flink run examples/gelly/flink-gelly-examples_*.jar \ --algorithm GraphMetrics --order directed \ --input RMatGraph --type integer --scale 20 --simplify directed \ --output print -~~~ +{% endhighlight %} The size of the graph is adjusted by the *\-\-scale* and *\-\-edge_factor* parameters. The [library generator](./graph_generators.html#rmat-graph) provides access to additional configuration to adjust the @@ -114,7 +114,7 @@ Sample social network data is provided by the [Stanford Network Analysis Project The [com-lj](http://snap.stanford.edu/data/bigdata/communities/com-lj.ungraph.txt.gz) data set is a good starter size. Run a few algorithms and monitor the job progress in Flink's Web UI: -~~~bash +{% highlight bash %} wget -O - http://snap.stanford.edu/data/bigdata/communities/com-lj.ungraph.txt.gz | gunzip -c > com-lj.ungraph.txt ./bin/flink run -q examples/gelly/flink-gelly-examples_*.jar \ @@ -131,7 +131,7 @@ wget -O - http://snap.stanford.edu/data/bigdata/communities/com-lj.ungraph.txt.g --algorithm JaccardIndex \ --input CSV --type integer --simplify undirected --input_filename com-lj.ungraph.txt --input_field_delimiter $'\t' \ --output hash -~~~ +{% endhighlight %} Please submit feature requests and report issues on the user [mailing list](https://flink.apache.org/community.html#mailing-lists) or [Flink Jira](https://issues.apache.org/jira/browse/FLINK). We welcome suggestions for new algorithms and features as diff --git a/docs/dev/libs/ml/cross_validation.md b/docs/dev/libs/ml/cross_validation.md index e2642829833aa4..01b01f94c6f7d5 100644 --- a/docs/dev/libs/ml/cross_validation.md +++ b/docs/dev/libs/ml/cross_validation.md @@ -62,7 +62,7 @@ In a *k-fold* strategy, the DataSet is split into *k* equal subsets. Then for ea For each training set, an algorithm is trained and then is evaluated based on the predictions based on the associated testing set. When an algorithm that has consistent grades (e.g. prediction errors) across held out datasets we can have some confidence that our approach (e.g. choice of algorithm / algorithm parameters / number of iterations) is robust against overfitting. -K-Fold Cross Validatation +K-Fold Cross Validation ### Multi-Random Splits diff --git a/docs/dev/libs/ml/index.md b/docs/dev/libs/ml/index.md index fbe3dce3dc6111..f8a75a45e50e9c 100644 --- a/docs/dev/libs/ml/index.md +++ b/docs/dev/libs/ml/index.md @@ -146,3 +146,5 @@ If one wants to chain a `Predictor` to a `Transformer` or a set of chained `Tran The Flink community welcomes all contributors who want to get involved in the development of Flink and its libraries. In order to get quickly started with contributing to FlinkML, please read our official [contribution guide]({{site.baseurl}}/dev/libs/ml/contribution_guide.html). + +{% top %} diff --git a/docs/dev/libs/ml/optimization.md b/docs/dev/libs/ml/optimization.md index 1e3bd2ae1db4e3..5ccde250730b7a 100644 --- a/docs/dev/libs/ml/optimization.md +++ b/docs/dev/libs/ml/optimization.md @@ -225,9 +225,9 @@ The loss function which is minimized has to implement the `LossFunction` interfa Either one defines ones own `LossFunction` or one uses the `GenericLossFunction` class which constructs the loss function from an outer loss function and a prediction function. An example can be seen here -```Scala +{% highlight scala %} val lossFunction = GenericLossFunction(SquaredLoss, LinearPrediction) -``` +{% endhighlight %} The full list of supported outer loss functions can be found [here](#partial-loss-function-values). The full list of supported prediction functions can be found [here](#prediction-function-values). diff --git a/docs/dev/libs/ml/quickstart.md b/docs/dev/libs/ml/quickstart.md index ea6f80497553dc..2e9a7b9505c9c4 100644 --- a/docs/dev/libs/ml/quickstart.md +++ b/docs/dev/libs/ml/quickstart.md @@ -129,15 +129,14 @@ and the [test set here](http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/b This is an astroparticle binary classification dataset, used by Hsu et al. [[3]](#hsu) in their practical Support Vector Machine (SVM) guide. It contains 4 numerical features, and the class label. -We can simply import the dataset then using: +We can simply import the dataset using: {% highlight scala %} import org.apache.flink.ml.MLUtils -val astroTrain: DataSet[LabeledVector] = MLUtils.readLibSVM(env, "/path/to/svmguide1") -val astroTest: DataSet[(Vector, Double)] = MLUtils.readLibSVM(env, "/path/to/svmguide1.t") - .map(x => (x.vector, x.label)) +val astroTrainLibSVM: DataSet[LabeledVector] = MLUtils.readLibSVM(env, "/path/to/svmguide1") +val astroTestLibSVM: DataSet[LabeledVector] = MLUtils.readLibSVM(env, "/path/to/svmguide1.t") {% endhighlight %} @@ -146,7 +145,25 @@ create a classifier. ## Classification -Once we have imported the dataset we can train a `Predictor` such as a linear SVM classifier. +After importing the training and test dataset, they need to be prepared for the classification. +Since Flink SVM only supports threshold binary values of `+1.0` and `-1.0`, a conversion is +needed after loading the LibSVM dataset because it is labelled using `1`s and `0`s. + +A conversion can be done using a simple normalizer mapping function: + +{% highlight scala %} + +import org.apache.flink.ml.math.Vector + +def normalizer : LabeledVector => LabeledVector = { + lv => LabeledVector(if (lv.label > 0.0) 1.0 else -1.0, lv.vector) +} +val astroTrain: DataSet[LabeledVector] = astroTrainLibSVM.map(normalizer) +val astroTest: DataSet[(Vector, Double)] = astroTestLibSVM.map(normalizer).map(x => (x.vector, x.label)) + +{% endhighlight %} + +Once we have converted the dataset we can train a `Predictor` such as a linear SVM classifier. We can set a number of parameters for the classifier. Here we set the `Blocks` parameter, which is used to split the input by the underlying CoCoA algorithm [[2]](#jaggi) uses. The regularization parameter determines the amount of $l_2$ regularization applied, which is used diff --git a/docs/dev/libs/storm_compatibility.md b/docs/dev/libs/storm_compatibility.md deleted file mode 100644 index 853b8e119ceaff..00000000000000 --- a/docs/dev/libs/storm_compatibility.md +++ /dev/null @@ -1,289 +0,0 @@ ---- -title: "Storm Compatibility" -is_beta: true -nav-parent_id: libs -nav-pos: 2 ---- - - -[Flink streaming]({{ site.baseurl }}/dev/datastream_api.html) is compatible with Apache Storm interfaces and therefore allows -reusing code that was implemented for Storm. - -You can: - -- execute a whole Storm `Topology` in Flink. -- use Storm `Spout`/`Bolt` as source/operator in Flink streaming programs. - -This document shows how to use existing Storm code with Flink. - -* This will be replaced by the TOC -{:toc} - -# Project Configuration - -Support for Storm is contained in the `flink-storm` Maven module. -The code resides in the `org.apache.flink.storm` package. - -Add the following dependency to your `pom.xml` if you want to execute Storm code in Flink. - -~~~xml - - org.apache.flink - flink-storm{{ site.scala_version_suffix }} - {{site.version}} - -~~~ - -**Please note**: Do not add `storm-core` as a dependency. It is already included via `flink-storm`. - -**Please note**: `flink-storm` is not part of the provided binary Flink distribution. -Thus, you need to include `flink-storm` classes (and their dependencies) in your program jar (also called uber-jar or fat-jar) that is submitted to Flink's JobManager. -See *WordCount Storm* within `flink-storm-examples/pom.xml` for an example how to package a jar correctly. - -If you want to avoid large uber-jars, you can manually copy `storm-core-0.9.4.jar`, `json-simple-1.1.jar` and `flink-storm-{{site.version}}.jar` into Flink's `lib/` folder of each cluster node (*before* the cluster is started). -For this case, it is sufficient to include only your own Spout and Bolt classes (and their internal dependencies) into the program jar. - -# Execute Storm Topologies - -Flink provides a Storm compatible API (`org.apache.flink.storm.api`) that offers replacements for the following classes: - -- `StormSubmitter` replaced by `FlinkSubmitter` -- `NimbusClient` and `Client` replaced by `FlinkClient` -- `LocalCluster` replaced by `FlinkLocalCluster` - -In order to submit a Storm topology to Flink, it is sufficient to replace the used Storm classes with their Flink replacements in the Storm *client code that assembles* the topology. -The actual runtime code, ie, Spouts and Bolts, can be used *unmodified*. -If a topology is executed in a remote cluster, parameters `nimbus.host` and `nimbus.thrift.port` are used as `jobmanger.rpc.address` and `jobmanger.rpc.port`, respectively. If a parameter is not specified, the value is taken from `flink-conf.yaml`. - -

    -
    -~~~java -TopologyBuilder builder = new TopologyBuilder(); // the Storm topology builder - -// actual topology assembling code and used Spouts/Bolts can be used as-is -builder.setSpout("source", new FileSpout(inputFilePath)); -builder.setBolt("tokenizer", new BoltTokenizer()).shuffleGrouping("source"); -builder.setBolt("counter", new BoltCounter()).fieldsGrouping("tokenizer", new Fields("word")); -builder.setBolt("sink", new BoltFileSink(outputFilePath)).shuffleGrouping("counter"); - -Config conf = new Config(); -if(runLocal) { // submit to test cluster - // replaces: LocalCluster cluster = new LocalCluster(); - FlinkLocalCluster cluster = new FlinkLocalCluster(); - cluster.submitTopology("WordCount", conf, FlinkTopology.createTopology(builder)); -} else { // submit to remote cluster - // optional - // conf.put(Config.NIMBUS_HOST, "remoteHost"); - // conf.put(Config.NIMBUS_THRIFT_PORT, 6123); - // replaces: StormSubmitter.submitTopology(topologyId, conf, builder.createTopology()); - FlinkSubmitter.submitTopology("WordCount", conf, FlinkTopology.createTopology(builder)); -} -~~~ -
    -
    - -# Embed Storm Operators in Flink Streaming Programs - -As an alternative, Spouts and Bolts can be embedded into regular streaming programs. -The Storm compatibility layer offers a wrapper classes for each, namely `SpoutWrapper` and `BoltWrapper` (`org.apache.flink.storm.wrappers`). - -Per default, both wrappers convert Storm output tuples to Flink's [Tuple]({{site.baseurl}}/dev/api_concepts.html#tuples-and-case-classes) types (ie, `Tuple0` to `Tuple25` according to the number of fields of the Storm tuples). -For single field output tuples a conversion to the field's data type is also possible (eg, `String` instead of `Tuple1`). - -Because Flink cannot infer the output field types of Storm operators, it is required to specify the output type manually. -In order to get the correct `TypeInformation` object, Flink's `TypeExtractor` can be used. - -## Embed Spouts - -In order to use a Spout as Flink source, use `StreamExecutionEnvironment.addSource(SourceFunction, TypeInformation)`. -The Spout object is handed to the constructor of `SpoutWrapper` that serves as first argument to `addSource(...)`. -The generic type declaration `OUT` specifies the type of the source output stream. - -
    -
    -~~~java -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - -// stream has `raw` type (single field output streams only) -DataStream rawInput = env.addSource( - new SpoutWrapper(new FileSpout(localFilePath), new String[] { Utils.DEFAULT_STREAM_ID }), // emit default output stream as raw type - TypeExtractor.getForClass(String.class)); // output type - -// process data stream -[...] -~~~ -
    -
    - -If a Spout emits a finite number of tuples, `SpoutWrapper` can be configures to terminate automatically by setting `numberOfInvocations` parameter in its constructor. -This allows the Flink program to shut down automatically after all data is processed. -Per default the program will run until it is [canceled]({{site.baseurl}}/ops/cli.html) manually. - - -## Embed Bolts - -In order to use a Bolt as Flink operator, use `DataStream.transform(String, TypeInformation, OneInputStreamOperator)`. -The Bolt object is handed to the constructor of `BoltWrapper` that serves as last argument to `transform(...)`. -The generic type declarations `IN` and `OUT` specify the type of the operator's input and output stream, respectively. - -
    -
    -~~~java -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); -DataStream text = env.readTextFile(localFilePath); - -DataStream> counts = text.transform( - "tokenizer", // operator name - TypeExtractor.getForObject(new Tuple2("", 0)), // output type - new BoltWrapper>(new BoltTokenizer())); // Bolt operator - -// do further processing -[...] -~~~ -
    -
    - -### Named Attribute Access for Embedded Bolts - -Bolts can accesses input tuple fields via name (additionally to access via index). -To use this feature with embedded Bolts, you need to have either a - - 1. [POJO]({{site.baseurl}}/dev/api_concepts.html#pojos) type input stream or - 2. [Tuple]({{site.baseurl}}/dev/api_concepts.html#tuples-and-case-classes) type input stream and specify the input schema (i.e. name-to-index-mapping) - -For POJO input types, Flink accesses the fields via reflection. -For this case, Flink expects either a corresponding public member variable or public getter method. -For example, if a Bolt accesses a field via name `sentence` (eg, `String s = input.getStringByField("sentence");`), the input POJO class must have a member variable `public String sentence;` or method `public String getSentence() { ... };` (pay attention to camel-case naming). - -For `Tuple` input types, it is required to specify the input schema using Storm's `Fields` class. -For this case, the constructor of `BoltWrapper` takes an additional argument: `new BoltWrapper, ...>(..., new Fields("sentence"))`. -The input type is `Tuple1` and `Fields("sentence")` specify that `input.getStringByField("sentence")` is equivalent to `input.getString(0)`. - -See [BoltTokenizerWordCountPojo](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java) and [BoltTokenizerWordCountWithNames](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java) for examples. - -## Configuring Spouts and Bolts - -In Storm, Spouts and Bolts can be configured with a globally distributed `Map` object that is given to `submitTopology(...)` method of `LocalCluster` or `StormSubmitter`. -This `Map` is provided by the user next to the topology and gets forwarded as a parameter to the calls `Spout.open(...)` and `Bolt.prepare(...)`. -If a whole topology is executed in Flink using `FlinkTopologyBuilder` etc., there is no special attention required – it works as in regular Storm. - -For embedded usage, Flink's configuration mechanism must be used. -A global configuration can be set in a `StreamExecutionEnvironment` via `.getConfig().setGlobalJobParameters(...)`. -Flink's regular `Configuration` class can be used to configure Spouts and Bolts. -However, `Configuration` does not support arbitrary key data types as Storm does (only `String` keys are allowed). -Thus, Flink additionally provides `StormConfig` class that can be used like a raw `Map` to provide full compatibility to Storm. - -
    -
    -~~~java -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - -StormConfig config = new StormConfig(); -// set config values -[...] - -// set global Storm configuration -env.getConfig().setGlobalJobParameters(config); - -// assemble program with embedded Spouts and/or Bolts -[...] -~~~ -
    -
    - -## Multiple Output Streams - -Flink can also handle the declaration of multiple output streams for Spouts and Bolts. -If a whole topology is executed in Flink using `FlinkTopologyBuilder` etc., there is no special attention required – it works as in regular Storm. - -For embedded usage, the output stream will be of data type `SplitStreamType` and must be split by using `DataStream.split(...)` and `SplitStream.select(...)`. -Flink provides the predefined output selector `StormStreamSelector` for `.split(...)` already. -Furthermore, the wrapper type `SplitStreamTuple` can be removed using `SplitStreamMapper`. - -
    -
    -~~~java -[...] - -// get DataStream from Spout or Bolt which declares two output streams s1 and s2 with output type SomeType -DataStream> multiStream = ... - -SplitStream> splitStream = multiStream.split(new StormStreamSelector()); - -// remove SplitStreamType using SplitStreamMapper to get data stream of type SomeType -DataStream s1 = splitStream.select("s1").map(new SplitStreamMapper()).returns(SomeType.class); -DataStream s2 = splitStream.select("s2").map(new SplitStreamMapper()).returns(SomeType.class); - -// do further processing on s1 and s2 -[...] -~~~ -
    -
    - -See [SpoutSplitExample.java](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java) for a full example. - -# Flink Extensions - -## Finite Spouts - -In Flink, streaming sources can be finite, ie, emit a finite number of records and stop after emitting the last record. However, Spouts usually emit infinite streams. -The bridge between the two approaches is the `FiniteSpout` interface which, in addition to `IRichSpout`, contains a `reachedEnd()` method, where the user can specify a stopping-condition. -The user can create a finite Spout by implementing this interface instead of (or additionally to) `IRichSpout`, and implementing the `reachedEnd()` method in addition. -In contrast to a `SpoutWrapper` that is configured to emit a finite number of tuples, `FiniteSpout` interface allows to implement more complex termination criteria. - -Although finite Spouts are not necessary to embed Spouts into a Flink streaming program or to submit a whole Storm topology to Flink, there are cases where they may come in handy: - - * to achieve that a native Spout behaves the same way as a finite Flink source with minimal modifications - * the user wants to process a stream only for some time; after that, the Spout can stop automatically - * reading a file into a stream - * for testing purposes - -An example of a finite Spout that emits records for 10 seconds only: - -
    -
    -~~~java -public class TimedFiniteSpout extends BaseRichSpout implements FiniteSpout { - [...] // implement open(), nextTuple(), ... - - private long starttime = System.currentTimeMillis(); - - public boolean reachedEnd() { - return System.currentTimeMillis() - starttime > 10000l; - } -} -~~~ -
    -
    - -# Storm Compatibility Examples - -You can find more examples in Maven module `flink-storm-examples`. -For the different versions of WordCount, see [README.md](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/README.md). -To run the examples, you need to assemble a correct jar file. -`flink-storm-examples-{{ site.version }}.jar` is **no** valid jar file for job execution (it is only a standard maven artifact). - -There are example jars for embedded Spout and Bolt, namely `WordCount-SpoutSource.jar` and `WordCount-BoltTokenizer.jar`, respectively. -Compare `pom.xml` to see how both jars are built. -Furthermore, there is one example for whole Storm topologies (`WordCount-StormTopology.jar`). - -You can run each of those examples via `bin/flink run .jar`. The correct entry point class is contained in each jar's manifest file. - -{% top %} diff --git a/docs/dev/linking.md b/docs/dev/linking.md deleted file mode 100644 index 78ef54494831f4..00000000000000 --- a/docs/dev/linking.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -nav-title: "Linking with Optional Modules" -title: "Linking with modules not contained in the binary distribution" -nav-parent_id: start -nav-pos: 10 ---- - - -The binary distribution contains jar packages in the `lib` folder that are automatically -provided to the classpath of your distributed programs. Almost all of Flink classes are -located there with a few exceptions, for example the streaming connectors and some freshly -added modules. To run code depending on these modules you need to make them accessible -during runtime, for which we suggest two options: - -1. Either copy the required jar files to the `lib` folder onto all of your TaskManagers. -Note that you have to restart your TaskManagers after this. -2. Or package them with your code. - -The latter version is recommended as it respects the classloader management in Flink. - -### Packaging dependencies with your usercode with Maven - -To provide these dependencies not included by Flink we suggest two options with Maven. - -1. The maven assembly plugin builds a so-called uber-jar (executable jar) containing all your dependencies. -The assembly configuration is straight-forward, but the resulting jar might become bulky. -See [maven-assembly-plugin](http://maven.apache.org/plugins/maven-assembly-plugin/usage.html) for further information. -2. The maven unpack plugin unpacks the relevant parts of the dependencies and -then packages it with your code. - -Using the latter approach in order to bundle the Kafka connector, `flink-connector-kafka` -you would need to add the classes from both the connector and the Kafka API itself. Add -the following to your plugins section. - -~~~xml - - org.apache.maven.plugins - maven-dependency-plugin - 2.9 - - - unpack - - prepare-package - - unpack - - - - - - org.apache.flink - flink-connector-kafka - {{ site.version }} - jar - false - ${project.build.directory}/classes - org/apache/flink/** - - - - org.apache.kafka - kafka_ - - jar - false - ${project.build.directory}/classes - kafka/** - - - - - - -~~~ - -Now when running `mvn clean package` the produced jar includes the required dependencies. - -{% top %} diff --git a/docs/dev/linking_with_flink.md b/docs/dev/linking_with_flink.md deleted file mode 100644 index f2380b23a07f68..00000000000000 --- a/docs/dev/linking_with_flink.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -title: "Linking with Flink" -nav-parent_id: start -nav-pos: 2 ---- - - -To write programs with Flink, you need to include the Flink library corresponding to -your programming language in your project. - -The simplest way to do this is to use one of the quickstart scripts: either for -[Java]({{ site.baseurl }}/quickstart/java_api_quickstart.html) or for [Scala]({{ site.baseurl }}/quickstart/scala_api_quickstart.html). They -create a blank project from a template (a Maven Archetype), which sets up everything for you. To -manually create the project, you can use the archetype and create a project by calling: - -
    -
    -{% highlight bash %} -mvn archetype:generate \ - -DarchetypeGroupId=org.apache.flink \ - -DarchetypeArtifactId=flink-quickstart-java \ - -DarchetypeVersion={{site.version }} -{% endhighlight %} -
    -
    -{% highlight bash %} -mvn archetype:generate \ - -DarchetypeGroupId=org.apache.flink \ - -DarchetypeArtifactId=flink-quickstart-scala \ - -DarchetypeVersion={{site.version }} -{% endhighlight %} -
    -
    - -The archetypes are working for stable releases and preview versions (`-SNAPSHOT`). - -If you want to add Flink to an existing Maven project, add the following entry to your -*dependencies* section in the *pom.xml* file of your project: - -
    -
    -{% highlight xml %} - - - org.apache.flink - flink-streaming-java{{ site.scala_version_suffix }} - {{site.version }} - - - - org.apache.flink - flink-java - {{site.version }} - - - org.apache.flink - flink-clients{{ site.scala_version_suffix }} - {{site.version }} - -{% endhighlight %} -
    -
    -{% highlight xml %} - - - org.apache.flink - flink-streaming-scala{{ site.scala_version_suffix }} - {{site.version }} - - - - org.apache.flink - flink-scala{{ site.scala_version_suffix }} - {{site.version }} - - - org.apache.flink - flink-clients{{ site.scala_version_suffix }} - {{site.version }} - -{% endhighlight %} - -**Important:** When working with the Scala API you must have one of these two imports: -{% highlight scala %} -import org.apache.flink.api.scala._ -{% endhighlight %} - -or - -{% highlight scala %} -import org.apache.flink.api.scala.createTypeInformation -{% endhighlight %} - -The reason is that Flink analyzes the types that are used in a program and generates serializers -and comparators for them. By having either of those imports you enable an implicit conversion -that creates the type information for Flink operations. - -If you would rather use SBT, see [here]({{ site.baseurl }}/quickstart/scala_api_quickstart.html#sbt). -
    -
    - -#### Scala Dependency Versions - -Because Scala 2.10 binary is not compatible with Scala 2.11 binary, we provide multiple artifacts -to support both Scala versions. - -Starting from the 0.10 line, we cross-build all Flink modules for both 2.10 and 2.11. If you want -to run your program on Flink with Scala 2.11, you need to add a `_2.11` suffix to the `artifactId` -values of the Flink modules in your dependencies section. - -If you are looking for building Flink with Scala 2.11, please check -[build guide]({{ site.baseurl }}/start/building.html#scala-versions). - -#### Hadoop Dependency Versions - -If you are using Flink together with Hadoop, the version of the dependency may vary depending on the -version of Hadoop (or more specifically, HDFS) that you want to use Flink with. Please refer to the -[downloads page](http://flink.apache.org/downloads.html) for a list of available versions, and instructions -on how to link with custom versions of Hadoop. - -In order to link against the latest SNAPSHOT versions of the code, please follow -[this guide](http://flink.apache.org/how-to-contribute.html#snapshots-nightly-builds). - -The *flink-clients* dependency is only necessary to invoke the Flink program locally (for example to -run it standalone for testing and debugging). If you intend to only export the program as a JAR -file and [run it on a cluster]({{ site.baseurl }}/dev/cluster_execution.html), you can skip that dependency. - -{% top %} - diff --git a/docs/dev/local_execution.md b/docs/dev/local_execution.md index 326d5157fe3a78..d03029b90b4da1 100644 --- a/docs/dev/local_execution.md +++ b/docs/dev/local_execution.md @@ -43,13 +43,13 @@ Please also refer to the [debugging section]({{ site.baseurl }}/dev/batch/index. If you are developing your program in a Maven project, you have to add the `flink-clients` module using this dependency: -~~~xml +{% highlight xml %} org.apache.flink flink-clients{{ site.scala_version_suffix }} {{site.version}} -~~~ +{% endhighlight %} ## Local Environment @@ -59,7 +59,7 @@ The local environment is instantiated via the method `ExecutionEnvironment.creat In most cases, calling `ExecutionEnvironment.getExecutionEnvironment()` is the even better way to go. That method returns a `LocalEnvironment` when the program is started locally (outside the command line interface), and it returns a pre-configured environment for cluster execution, when the program is invoked by the [command line interface]({{ site.baseurl }}/ops/cli.html). -~~~java +{% highlight java %} public static void main(String[] args) throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(); @@ -75,17 +75,17 @@ public static void main(String[] args) throws Exception { JobExecutionResult res = env.execute(); } -~~~ +{% endhighlight %} The `JobExecutionResult` object, which is returned after the execution finished, contains the program runtime and the accumulator results. The `LocalEnvironment` allows also to pass custom configuration values to Flink. -~~~java +{% highlight java %} Configuration conf = new Configuration(); conf.setFloat(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, 0.5f); final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf); -~~~ +{% endhighlight %} *Note:* The local execution environments do not start any web frontend to monitor the execution. @@ -97,7 +97,7 @@ Users can use algorithms implemented for batch processing also for cases that ar **Skeleton for Collection-based execution** -~~~java +{% highlight java %} public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = new CollectionEnvironment(); @@ -118,7 +118,7 @@ public static void main(String[] args) throws Exception { System.err.println("Result = "+t); } } -~~~ +{% endhighlight %} The `flink-examples-batch` module contains a full example, called `CollectionExecutionExample`. diff --git a/docs/dev/migration.md b/docs/dev/migration.md index b9ed8fbbdd5491..d472653cbb38d0 100644 --- a/docs/dev/migration.md +++ b/docs/dev/migration.md @@ -25,6 +25,17 @@ under the License. * This will be replaced by the TOC {:toc} +## Migrating from Flink 1.3+ to Flink 1.7 + +### API changes for serializer snapshots + +This would be relevant mostly for users implementing custom `TypeSerializer`s for their state. + +The old `TypeSerializerConfigSnapshot` abstraction is now deprecated, and will be fully removed in the future +in favor of the new `TypeSerializerSnapshot`. For details and guides on how to migrate, please see +[Migrating from deprecated serializer snapshot APIs before Flink 1.7] +({{ site.baseurl }}/dev/stream/state/custom_serialization.html#migration-from-deprecated-serializer-snapshot-apis-before-Flink-1.7). + ## Migrating from Flink 1.2 to Flink 1.3 There are a few APIs that have been changed since Flink 1.2. Most of the changes are documented in their @@ -59,7 +70,7 @@ now clean of specific logger dependencies. Example and quickstart archetypes already have loggers specified and should not be affected. For other custom projects, make sure to add logger dependencies. For example, in Maven's `pom.xml`, you can add: -~~~xml +{% highlight xml %} org.slf4j slf4j-log4j12 @@ -71,7 +82,7 @@ For other custom projects, make sure to add logger dependencies. For example, in log4j 1.2.17 -~~~ +{% endhighlight %} ## Migrating from Flink 1.1 to Flink 1.2 diff --git a/docs/dev/projectsetup/dependencies.md b/docs/dev/projectsetup/dependencies.md new file mode 100644 index 00000000000000..447a071f2c53db --- /dev/null +++ b/docs/dev/projectsetup/dependencies.md @@ -0,0 +1,244 @@ +--- +title: "Configuring Dependencies, Connectors, Libraries" +nav-parent_id: projectsetup +nav-pos: 2 +--- + + +Every Flink application depends on a set of Flink libraries. At the bare minimum, the application depends +on the Flink APIs. Many applications depend in addition on certain connector libraries (like Kafka, Cassandra, etc.). +When running Flink applications (either in a distributed deployment, or in the IDE for testing), the Flink +runtime library must be available as well. + + +## Flink Core and Application Dependencies + +As with most systems that run user-defined applications, there are two broad categories of dependencies and libraries in Flink: + + - **Flink Core Dependencies**: Flink itself consists of a set of classes and dependencies that are needed to run the system, for example + coordination, networking, checkpoints, failover, APIs, operations (such as windowing), resource management, etc. + The set of all these classes and dependencies forms the core of Flink's runtime and must be present when a Flink + application is started. + + These core classes and dependencies are packaged in the `flink-dist` jar. They are part of Flink's `lib` folder and + part of the basic Flink container images. Think of these dependencies as similar to Java's core library (`rt.jar`, `charsets.jar`, etc.), + which contains the classes like `String` and `List`. + + The Flink Core Dependencies do not contain any connectors or libraries (CEP, SQL, ML, etc.) in order to avoid having an excessive + number of dependencies and classes in the classpath by default. In fact, we try to keep the core dependencies as slim as possible + to keep the default classpath small and avoid dependency clashes. + + - The **User Application Dependencies** are all connectors, formats, or libraries that a specific user application needs. + + The user application is typically packaged into an *application jar*, which contains the application code and the required + connector and library dependencies. + + The user application dependencies explicitly do not include the Flink DataSet / DataStream APIs and runtime dependencies, + because those are already part of Flink's Core Dependencies. + + +## Setting up a Project: Basic Dependencies + +Every Flink application needs as the bare minimum the API dependencies, to develop against. +For Maven, you can use the [Java Project Template]({{ site.baseurl }}/dev/projectsetup/java_api_quickstart.html) +or [Scala Project Template]({{ site.baseurl }}/dev/projectsetup/scala_api_quickstart.html) to create +a program skeleton with these initial dependencies. + +When setting up a project manually, you need to add the following dependencies for the Java/Scala API +(here presented in Maven syntax, but the same dependencies apply to other build tools (Gradle, SBT, etc.) as well. + +
    +
    +{% highlight xml %} + + org.apache.flink + flink-java + {{site.version }} + provided + + + org.apache.flink + flink-streaming-java{{ site.scala_version_suffix }} + {{site.version }} + provided + +{% endhighlight %} +
    +
    +{% highlight xml %} + + org.apache.flink + flink-scala{{ site.scala_version_suffix }} + {{site.version }} + provided + + + org.apache.flink + flink-streaming-scala{{ site.scala_version_suffix }} + {{site.version }} + provided + +{% endhighlight %} +
    +
    + +**Important:** Please note that all these dependencies have their scope set to *provided*. +That means that they are needed to compile against, but that they should not be packaged into the +project's resulting application jar file - these dependencies are Flink Core Dependencies, +which are already available in any setup. + +It is highly recommended to keep the dependencies in scope *provided*. If they are not set to *provided*, +the best case is that the resulting JAR becomes excessively large, because it also contains all Flink core +dependencies. The worst case is that the Flink core dependencies that are added to the application's jar file +clash with some of your own dependency versions (which is normally avoided through inverted classloading). + +**Note on IntelliJ:** To make the applications run within IntelliJ IDEA, the Flink dependencies need +to be declared in scope *compile* rather than *provided*. Otherwise IntelliJ will not add them to the classpath and +the in-IDE execution will fail with a `NoClassDefFountError`. To avoid having to declare the +dependency scope as *compile* (which is not recommended, see above), the above linked Java- and Scala +project templates use a trick: They add a profile that selectively activates when the application +is run in IntelliJ and only then promotes the dependencies to scope *compile*, without affecting +the packaging of the JAR files. + + +## Adding Connector and Library Dependencies + +Most applications need specific connectors or libraries to run, for example a connector to Kafka, Cassandra, etc. +These connectors are not part of Flink's core dependencies and must hence be added as dependencies to the application + +Below is an example adding the connector for Kafka 0.10 as a dependency (Maven syntax): +{% highlight xml %} + + org.apache.flink + flink-connector-kafka-0.10{{ site.scala_version_suffix }} + {{site.version }} + +{% endhighlight %} + +We recommend to package the application code and all its required dependencies into one *jar-with-dependencies* which +we refer to as the *application jar*. The application jar can be submitted to an already running Flink cluster, +or added to a Flink application container image. + +Projects created from the [Java Project Template]({{ site.baseurl }}/dev/projectsetup/java_api_quickstart.html) or +[Scala Project Template]({{ site.baseurl }}/dev/projectsetup/scala_api_quickstart.html) are configured to automatically include +the application dependencies into the application jar when running `mvn clean package`. For projects that are +not set up from those templates, we recommend to add the Maven Shade Plugin (as listed in the Appendix below) +to build the application jar with all required dependencies. + +**Important:** For Maven (and other build tools) to correctly package the dependencies into the application jar, +these application dependencies must be specified in scope *compile* (unlike the core dependencies, which +must be specified in scope *provided*). + + +## Scala Versions + +Scala versions (2.10, 2.11, 2.12, etc.) are not binary compatible with one another. +For that reason, Flink for Scala 2.11 cannot be used with an application that uses +Scala 2.12. + +All Flink dependencies that (transitively) depend on Scala are suffixed with the +Scala version that they are built for, for example `flink-streaming-scala_2.11`. + +Developers that only use Java can pick any Scala version, Scala developers need to +pick the Scala version that matches their application's Scala version. + +Please refer to the [build guide]({{ site.baseurl }}/flinkDev/building.html#scala-versions) +for details on how to build Flink for a specific Scala version. + +**Note:** Because of major breaking changes in Scala 2.12, Flink 1.5 currently builds only for Scala 2.11. +We aim to add support for Scala 2.12 in the next versions. + + +## Hadoop Dependencies + +**General rule: It should never be necessary to add Hadoop dependencies directly to your application.** +*(The only exception being when using existing Hadoop input-/output formats with Flink's Hadoop compatibility wrappers)* + +If you want to use Flink with Hadoop, you need to have a Flink setup that includes the Hadoop dependencies, rather than +adding Hadoop as an application dependency. Please refer to the [Hadoop Setup Guide]({{ site.baseurl }}/ops/deployment/hadoop.html) +for details. + +There are two main reasons for that design: + + - Some Hadoop interaction happens in Flink's core, possibly before the user application is started, for example + setting up HDFS for checkpoints, authenticating via Hadoop's Kerberos tokens, or deployment on YARN. + + - Flink's inverted classloading approach hides many transitive dependencies from the core dependencies. That applies not only + to Flink's own core dependencies, but also to Hadoop's dependencies when present in the setup. + That way, applications can use different versions of the same dependencies without running into dependency conflicts (and + trust us, that's a big deal, because Hadoops dependency tree is huge.) + +If you need Hadoop dependencies during testing or development inside the IDE (for example for HDFS access), please configure +these dependencies similar to the scope of the dependencies to *test* or to *provided*. + + +## Appendix: Template for building a Jar with Dependencies + +To build an application JAR that contains all dependencies required for declared connectors and libraries, +you can use the following shade plugin definition: + +{% highlight xml %} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + my.programs.main.clazz + + + + + + + + +{% endhighlight %} + +{% top %} + diff --git a/docs/dev/projectsetup/index.md b/docs/dev/projectsetup/index.md new file mode 100644 index 00000000000000..f92bcd9f279ebe --- /dev/null +++ b/docs/dev/projectsetup/index.md @@ -0,0 +1,25 @@ +--- +title: "Project Build Setup" +nav-id: projectsetup +nav-title: 'Project Build Setup' +nav-parent_id: dev +nav-pos: 0 +--- + diff --git a/docs/dev/projectsetup/java_api_quickstart.md b/docs/dev/projectsetup/java_api_quickstart.md new file mode 100644 index 00000000000000..fd2eef503790e8 --- /dev/null +++ b/docs/dev/projectsetup/java_api_quickstart.md @@ -0,0 +1,354 @@ +--- +title: "Project Template for Java" +nav-title: Project Template for Java +nav-parent_id: projectsetup +nav-pos: 0 +--- + + +* This will be replaced by the TOC +{:toc} + + +## Build Tools + +Flink projects can be built with different build tools. +In order to get started quickly, Flink provides project templates for the following build tools: + +- [Maven](#maven) +- [Gradle](#gradle) + +These templates help you to set up the project structure and to create the initial build files. + +## Maven + +### Requirements + +The only requirements are working __Maven 3.0.4__ (or higher) and __Java 8.x__ installations. + +### Create Project + +Use one of the following commands to __create a project__: + + +
    +
    + {% highlight bash %} + $ mvn archetype:generate \ + -DarchetypeGroupId=org.apache.flink \ + -DarchetypeArtifactId=flink-quickstart-java \{% unless site.is_stable %} + -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/ \{% endunless %} + -DarchetypeVersion={{site.version}} + {% endhighlight %} + This allows you to name your newly created project. It will interactively ask you for the groupId, artifactId, and package name. +
    +
    + {% highlight bash %} +{% if site.is_stable %} + $ curl https://flink.apache.org/q/quickstart.sh | bash -s {{site.version}} +{% else %} + $ curl https://flink.apache.org/q/quickstart-SNAPSHOT.sh | bash -s {{site.version}} +{% endif %} + {% endhighlight %} + +
    + {% unless site.is_stable %} +

    + Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the command line. If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For details about this change, please refer to Maven official document +

    + {% endunless %} +
    + +### Inspect Project + +There will be a new directory in your working directory. If you've used +the _curl_ approach, the directory is called `quickstart`. Otherwise, +it has the name of your `artifactId`: + +{% highlight bash %} +$ tree quickstart/ +quickstart/ +├── pom.xml +└── src + └── main + ├── java + │   └── org + │   └── myorg + │   └── quickstart + │   ├── BatchJob.java + │   └── StreamingJob.java + └── resources + └── log4j.properties +{% endhighlight %} + +The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. + +We recommend you __import this project into your IDE__ to develop and +test it. IntelliJ IDEA supports Maven projects out of the box. +If you use Eclipse, the [m2e plugin](http://www.eclipse.org/m2e/) +allows to [import Maven projects](http://books.sonatype.com/m2eclipse-book/reference/creating-sect-importing-projects.html#fig-creating-import). +Some Eclipse bundles include that plugin by default, others require you +to install it manually. + +*Please note*: The default JVM heapsize for Java may be too +small for Flink. You have to manually increase it. +In Eclipse, choose `Run Configurations -> Arguments` and write into the `VM Arguments` box: `-Xmx800m`. +In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. + +### Build Project + +If you want to __build/package your project__, go to your project directory and +run the '`mvn clean package`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencies to the application: `target/-.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. + +## Gradle + +### Requirements + +The only requirements are working __Gradle 3.x__ (or higher) and __Java 8.x__ installations. + +### Create Project + +Use one of the following commands to __create a project__: + + +
    +
    + + +
    + +
    + {% highlight gradle %} +buildscript { + repositories { + jcenter() // this applies only to the Gradle 'Shadow' plugin + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' + } +} + +plugins { + id 'java' + id 'application' + // shadow plugin to produce fat JARs + id 'com.github.johnrengelman.shadow' version '2.0.4' +} + + +// artifact properties +group = 'org.myorg.quickstart' +version = '0.1-SNAPSHOT' +mainClassName = 'org.myorg.quickstart.StreamingJob' +description = """Flink Quickstart Job""" + +ext { + javaVersion = '1.8' + flinkVersion = '{{ site.version }}' + scalaBinaryVersion = '{{ site.scala_version }}' + slf4jVersion = '1.7.7' + log4jVersion = '1.2.17' +} + + +sourceCompatibility = javaVersion +targetCompatibility = javaVersion +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] + +task wrapper(type: Wrapper) { + gradleVersion = '3.1' +} + +// declare where to find the dependencies of your project +repositories { + mavenCentral() + maven { url "https://repository.apache.org/content/repositories/snapshots/" } +} + +// NOTE: We cannot use "compileOnly" or "shadow" configurations since then we could not run code +// in the IDE or with "gradle run". We also cannot exclude transitive dependencies from the +// shadowJar yet (see https://github.com/johnrengelman/shadow/issues/159). +// -> Explicitly define the // libraries we want to be included in the "flinkShadowJar" configuration! +configurations { + flinkShadowJar // dependencies which go into the shadowJar + + // always exclude these (also from transitive dependencies) since they are provided by Flink + flinkShadowJar.exclude group: 'org.apache.flink', module: 'force-shading' + flinkShadowJar.exclude group: 'com.google.code.findbugs', module: 'jsr305' + flinkShadowJar.exclude group: 'org.slf4j' + flinkShadowJar.exclude group: 'log4j' +} + +// declare the dependencies for your production and test code +dependencies { + // -------------------------------------------------------------- + // Compile-time dependencies that should NOT be part of the + // shadow jar and are provided in the lib folder of Flink + // -------------------------------------------------------------- + compile "org.apache.flink:flink-java:${flinkVersion}" + compile "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" + + // -------------------------------------------------------------- + // Dependencies that should be part of the shadow jar, e.g. + // connectors. These must be in the flinkShadowJar configuration! + // -------------------------------------------------------------- + //flinkShadowJar "org.apache.flink:flink-connector-kafka-0.11_${scalaBinaryVersion}:${flinkVersion}" + + compile "log4j:log4j:${log4jVersion}" + compile "org.slf4j:slf4j-log4j12:${slf4jVersion}" + + // Add test dependencies here. + // testCompile "junit:junit:4.12" +} + +// make compileOnly dependencies available for tests: +sourceSets { + main.compileClasspath += configurations.flinkShadowJar + main.runtimeClasspath += configurations.flinkShadowJar + + test.compileClasspath += configurations.flinkShadowJar + test.runtimeClasspath += configurations.flinkShadowJar + + javadoc.classpath += configurations.flinkShadowJar +} + +run.classpath = sourceSets.main.runtimeClasspath + +jar { + manifest { + attributes 'Built-By': System.getProperty('user.name'), + 'Build-Jdk': System.getProperty('java.version') + } +} + +shadowJar { + configurations = [project.configurations.flinkShadowJar] +} + {% endhighlight %} +
    +
    + {% highlight gradle %} +rootProject.name = 'quickstart' + {% endhighlight %} +
    +
    +
    + +
    + {% highlight bash %} + bash -c "$(curl https://flink.apache.org/q/gradle-quickstart.sh)" -- {{site.version}} {{site.scala_version}} + {% endhighlight %} + This allows you to name your newly created project. It will interactively ask + you for the project name, organization (also used for the package name), project version, + Scala and Flink version. +
    +
    + +### Inspect Project + +There will be a new directory in your working directory based on the +project name you provided, e.g. for `quickstart`: + +{% highlight bash %} +$ tree quickstart/ +quickstart/ +├── README +├── build.gradle +├── settings.gradle +└── src + └── main + ├── java + │   └── org + │   └── myorg + │   └── quickstart + │   ├── BatchJob.java + │   └── StreamingJob.java + └── resources + └── log4j.properties +{% endhighlight %} + +The sample project is a __Gradle project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. + +We recommend you __import this project into your IDE__ to develop and +test it. IntelliJ IDEA supports Gradle projects after installing the `Gradle` plugin. +Eclipse does so via the [Eclipse Buildship](https://projects.eclipse.org/projects/tools.buildship) plugin +(make sure to specify a Gradle version >= 3.0 in the last step of the import wizard; the `shadow` plugin requires it). +You may also use [Gradle's IDE integration](https://docs.gradle.org/current/userguide/userguide.html#ide-integration) +to create project files from Gradle. + + +*Please note*: The default JVM heapsize for Java may be too +small for Flink. You have to manually increase it. +In Eclipse, choose `Run Configurations -> Arguments` and write into the `VM Arguments` box: `-Xmx800m`. +In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. + +### Build Project + +If you want to __build/package your project__, go to your project directory and +run the '`gradle clean shadowJar`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencies to the application: `build/libs/--all.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClassName` setting in the `build.gradle` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. + +## Next Steps + +Write your application! + +If you are writing a streaming application and you are looking for inspiration what to write, +take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/tutorials/datastream_api.html#writing-a-flink-program). + +If you are writing a batch processing application and you are looking for inspiration what to write, +take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html). + +For a complete overview over the APIs, have a look at the +[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and +[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections. + +[Here]({{ site.baseurl }}/tutorials/local_setup.html) you can find out how to run an application outside the IDE on a local cluster. + +If you have any trouble, ask on our +[Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/). +We are happy to provide help. + +{% top %} diff --git a/docs/dev/projectsetup/scala_api_quickstart.md b/docs/dev/projectsetup/scala_api_quickstart.md new file mode 100644 index 00000000000000..da38eb7cdd6219 --- /dev/null +++ b/docs/dev/projectsetup/scala_api_quickstart.md @@ -0,0 +1,230 @@ +--- +title: "Project Template for Scala" +nav-title: Project Template for Scala +nav-parent_id: projectsetup +nav-pos: 1 +--- + + +* This will be replaced by the TOC +{:toc} + + +## Build Tools + +Flink projects can be built with different build tools. +In order to get started quickly, Flink provides project templates for the following build tools: + +- [SBT](#sbt) +- [Maven](#maven) + +These templates help you to set up the project structure and to create the initial build files. + +## SBT + +### Create Project + +You can scaffold a new project via either of the following two methods: + + + +
    +
    + {% highlight bash %} + $ sbt new tillrohrmann/flink-project.g8 + {% endhighlight %} + This will prompt you for a couple of parameters (project name, flink version...) and then create a Flink project from the flink-project template. + You need sbt >= 0.13.13 to execute this command. You can follow this installation guide to obtain it if necessary. +
    +
    + {% highlight bash %} + $ bash <(curl https://flink.apache.org/q/sbt-quickstart.sh) + {% endhighlight %} + This will create a Flink project in the specified project directory. +
    +
    + +### Build Project + +In order to build your project you simply have to issue the `sbt clean assembly` command. +This will create the fat-jar __your-project-name-assembly-0.1-SNAPSHOT.jar__ in the directory __target/scala_your-major-scala-version/__. + +### Run Project + +In order to run your project you have to issue the `sbt run` command. + +Per default, this will run your job in the same JVM as `sbt` is running. +In order to run your job in a distinct JVM, add the following line to `build.sbt` + +{% highlight scala %} +fork in run := true +{% endhighlight %} + + +#### IntelliJ + +We recommend using [IntelliJ](https://www.jetbrains.com/idea/) for your Flink job development. +In order to get started, you have to import your newly created project into IntelliJ. +You can do this via `File -> New -> Project from Existing Sources...` and then choosing your project's directory. +IntelliJ will then automatically detect the `build.sbt` file and set everything up. + +In order to run your Flink job, it is recommended to choose the `mainRunner` module as the classpath of your __Run/Debug Configuration__. +This will ensure, that all dependencies which are set to _provided_ will be available upon execution. +You can configure the __Run/Debug Configurations__ via `Run -> Edit Configurations...` and then choose `mainRunner` from the _Use classpath of module_ dropbox. + +#### Eclipse + +In order to import the newly created project into [Eclipse](https://eclipse.org/), you first have to create Eclipse project files for it. +These project files can be created via the [sbteclipse](https://github.com/typesafehub/sbteclipse) plugin. +Add the following line to your `PROJECT_DIR/project/plugins.sbt` file: + +{% highlight bash %} +addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0") +{% endhighlight %} + +In `sbt` use the following command to create the Eclipse project files + +{% highlight bash %} +> eclipse +{% endhighlight %} + +Now you can import the project into Eclipse via `File -> Import... -> Existing Projects into Workspace` and then select the project directory. + +## Maven + +### Requirements + +The only requirements are working __Maven 3.0.4__ (or higher) and __Java 8.x__ installations. + + +### Create Project + +Use one of the following commands to __create a project__: + + + +
    +
    + {% highlight bash %} + $ mvn archetype:generate \ + -DarchetypeGroupId=org.apache.flink \ + -DarchetypeArtifactId=flink-quickstart-scala \{% unless site.is_stable %} + -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/ \{% endunless %} + -DarchetypeVersion={{site.version}} + {% endhighlight %} + This allows you to name your newly created project. It will interactively ask you for the groupId, artifactId, and package name. +
    +
    +{% highlight bash %} +{% if site.is_stable %} + $ curl https://flink.apache.org/q/quickstart-scala.sh | bash -s {{site.version}} +{% else %} + $ curl https://flink.apache.org/q/quickstart-scala-SNAPSHOT.sh | bash -s {{site.version}} +{% endif %} +{% endhighlight %} +
    + {% unless site.is_stable %} +

    + Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the commandline. If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For details about this change, please refer to Maven official document +

    + {% endunless %} +
    + + +### Inspect Project + +There will be a new directory in your working directory. If you've used +the _curl_ approach, the directory is called `quickstart`. Otherwise, +it has the name of your `artifactId`: + +{% highlight bash %} +$ tree quickstart/ +quickstart/ +├── pom.xml +└── src + └── main + ├── resources + │   └── log4j.properties + └── scala + └── org + └── myorg + └── quickstart + ├── BatchJob.scala + └── StreamingJob.scala +{% endhighlight %} + +The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. + +We recommend you __import this project into your IDE__. + +IntelliJ IDEA supports Maven out of the box and offers a plugin for Scala development. +From our experience, IntelliJ provides the best experience for developing Flink applications. + +For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites: + +* _Eclipse 4.x_ + * [Scala IDE](http://download.scala-ide.org/sdk/lithium/e44/scala211/stable/site) + * [m2eclipse-scala](http://alchim31.free.fr/m2e-scala/update-site) + * [Build Helper Maven Plugin](https://repo1.maven.org/maven2/.m2e/connectors/m2eclipse-buildhelper/0.15.0/N/0.15.0.201207090124/) +* _Eclipse 3.8_ + * [Scala IDE for Scala 2.11](http://download.scala-ide.org/sdk/helium/e38/scala211/stable/site) or [Scala IDE for Scala 2.10](http://download.scala-ide.org/sdk/helium/e38/scala210/stable/site) + * [m2eclipse-scala](http://alchim31.free.fr/m2e-scala/update-site) + * [Build Helper Maven Plugin](https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/) + +### Build Project + +If you want to __build/package your project__, go to your project directory and +run the '`mvn clean package`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencies to the application: `target/-.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink +can run time application from the JAR file without additionally specifying the main class. + + +## Next Steps + +Write your application! + +If you are writing a streaming application and you are looking for inspiration what to write, +take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/tutorials/datastream_api.html#writing-a-flink-program) + +If you are writing a batch processing application and you are looking for inspiration what to write, +take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html) + +For a complete overview over the APIa, have a look at the +[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and +[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections. + +[Here]({{ site.baseurl }}/tutorials/local_setup.html) you can find out how to run an application outside the IDE on a local cluster. + +If you have any trouble, ask on our +[Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/). +We are happy to provide help. + +{% top %} diff --git a/docs/dev/restart_strategies.md b/docs/dev/restart_strategies.md index a4901abf97b993..5be430e7f4804f 100644 --- a/docs/dev/restart_strategies.md +++ b/docs/dev/restart_strategies.md @@ -107,9 +107,9 @@ In-between two consecutive restart attempts, the restart strategy waits a fixed This strategy is enabled as default by setting the following configuration parameter in `flink-conf.yaml`. -~~~ +{% highlight yaml %} restart-strategy: fixed-delay -~~~ +{% endhighlight %} @@ -135,10 +135,10 @@ restart-strategy: fixed-delay For example: -~~~ +{% highlight yaml %} restart-strategy.fixed-delay.attempts: 3 restart-strategy.fixed-delay.delay: 10 s -~~~ +{% endhighlight %} The fixed delay restart strategy can also be set programmatically: @@ -172,9 +172,9 @@ In-between two consecutive restart attempts, the restart strategy waits a fixed This strategy is enabled as default by setting the following configuration parameter in `flink-conf.yaml`. -~~~ +{% highlight yaml %} restart-strategy: failure-rate -~~~ +{% endhighlight %}
    @@ -203,11 +203,11 @@ restart-strategy: failure-rate
    -~~~ +{% highlight yaml %} restart-strategy.failure-rate.max-failures-per-interval: 3 restart-strategy.failure-rate.failure-rate-interval: 5 min restart-strategy.failure-rate.delay: 10 s -~~~ +{% endhighlight %} The failure rate restart strategy can also be set programmatically: @@ -240,9 +240,9 @@ env.setRestartStrategy(RestartStrategies.failureRateRestart( The job fails directly and no restart is attempted. -~~~ +{% highlight yaml %} restart-strategy: none -~~~ +{% endhighlight %} The no restart strategy can also be set programmatically: @@ -264,7 +264,7 @@ env.setRestartStrategy(RestartStrategies.noRestart()) ### Fallback Restart Strategy The cluster defined restart strategy is used. -This helpful for streaming programs which enable checkpointing. -Per default, a fixed delay restart strategy is chosen if there is no other restart strategy defined. +This is helpful for streaming programs which enable checkpointing. +By default, a fixed delay restart strategy is chosen if there is no other restart strategy defined. {% top %} diff --git a/docs/dev/scala_shell.md b/docs/dev/scala_shell.md deleted file mode 100644 index b12060b3fe4c39..00000000000000 --- a/docs/dev/scala_shell.md +++ /dev/null @@ -1,195 +0,0 @@ ---- -title: "Scala REPL" -nav-parent_id: start -nav-pos: 5 ---- - - -Flink comes with an integrated interactive Scala Shell. -It can be used in a local setup as well as in a cluster setup. - -To use the shell with an integrated Flink cluster just execute: - -~~~bash -bin/start-scala-shell.sh local -~~~ - -in the root directory of your binary Flink directory. To run the Shell on a -cluster, please see the Setup section below. - -## Usage - -The shell supports Batch and Streaming. -Two different ExecutionEnvironments are automatically prebound after startup. -Use "benv" and "senv" to access the Batch and Streaming environment respectively. - -### DataSet API - -The following example will execute the wordcount program in the Scala shell: - -~~~scala -Scala-Flink> val text = benv.fromElements( - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles,") -Scala-Flink> val counts = text - .flatMap { _.toLowerCase.split("\\W+") } - .map { (_, 1) }.groupBy(0).sum(1) -Scala-Flink> counts.print() -~~~ - -The print() command will automatically send the specified tasks to the JobManager for execution and will show the result of the computation in the terminal. - -It is possible to write results to a file. However, in this case you need to call `execute`, to run your program: - -~~~scala -Scala-Flink> benv.execute("MyProgram") -~~~ - -### DataStream API - -Similar to the batch program above, we can execute a streaming program through the DataStream API: - -~~~scala -Scala-Flink> val textStreaming = senv.fromElements( - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles,") -Scala-Flink> val countsStreaming = textStreaming - .flatMap { _.toLowerCase.split("\\W+") } - .map { (_, 1) }.keyBy(0).sum(1) -Scala-Flink> countsStreaming.print() -Scala-Flink> senv.execute("Streaming Wordcount") -~~~ - -Note, that in the Streaming case, the print operation does not trigger execution directly. - -The Flink Shell comes with command history and auto-completion. - - -## Adding external dependencies - -It is possible to add external classpaths to the Scala-shell. These will be sent to the Jobmanager automatically alongside your shell program, when calling execute. - -Use the parameter `-a ` or `--addclasspath ` to load additional classes. - -~~~bash -bin/start-scala-shell.sh [local | remote | yarn] --addclasspath -~~~ - - -## Setup - -To get an overview of what options the Scala Shell provides, please use - -~~~bash -bin/start-scala-shell.sh --help -~~~ - -### Local - -To use the shell with an integrated Flink cluster just execute: - -~~~bash -bin/start-scala-shell.sh local -~~~ - - -### Remote - -To use it with a running cluster start the scala shell with the keyword `remote` -and supply the host and port of the JobManager with: - -~~~bash -bin/start-scala-shell.sh remote -~~~ - -### Yarn Scala Shell cluster - -The shell can deploy a Flink cluster to YARN, which is used exclusively by the -shell. The number of YARN containers can be controlled by the parameter `-n `. -The shell deploys a new Flink cluster on YARN and connects the -cluster. You can also specify options for YARN cluster such as memory for -JobManager, name of YARN application, etc. - -For example, to start a Yarn cluster for the Scala Shell with two TaskManagers -use the following: - -~~~bash - bin/start-scala-shell.sh yarn -n 2 -~~~ - -For all other options, see the full reference at the bottom. - - -### Yarn Session - -If you have previously deployed a Flink cluster using the Flink Yarn Session, -the Scala shell can connect with it using the following command: - -~~~bash - bin/start-scala-shell.sh yarn -~~~ - - -## Full Reference - -~~~bash -Flink Scala Shell -Usage: start-scala-shell.sh [local|remote|yarn] [options] ... - -Command: local [options] -Starts Flink scala shell with a local Flink cluster - -a | --addclasspath - Specifies additional jars to be used in Flink -Command: remote [options] -Starts Flink scala shell connecting to a remote cluster - - Remote host name as string - - Remote port as integer - - -a | --addclasspath - Specifies additional jars to be used in Flink -Command: yarn [options] -Starts Flink scala shell connecting to a yarn cluster - -n arg | --container arg - Number of YARN container to allocate (= Number of TaskManagers) - -jm arg | --jobManagerMemory arg - Memory for JobManager container [in MB] - -nm | --name - Set a custom name for the application on YARN - -qu | --queue - Specifies YARN queue - -s | --slots - Number of slots per TaskManager - -tm | --taskManagerMemory - Memory per TaskManager container [in MB] - -a | --addclasspath - Specifies additional jars to be used in Flink - --configDir - The configuration directory. - -h | --help - Prints this usage text -~~~ - -{% top %} diff --git a/docs/dev/stream/experimental.md b/docs/dev/stream/experimental.md index db029befb36bd2..f0a78a539126fb 100644 --- a/docs/dev/stream/experimental.md +++ b/docs/dev/stream/experimental.md @@ -77,3 +77,5 @@ Code example: .addSink(new DiscardingSink[Int]) env.execute() {% endhighlight %} + +{% top %} diff --git a/docs/dev/stream/operators/asyncio.md b/docs/dev/stream/operators/asyncio.md index 702d2ae993b1a9..e92e7a96d34e5e 100644 --- a/docs/dev/stream/operators/asyncio.md +++ b/docs/dev/stream/operators/asyncio.md @@ -104,17 +104,26 @@ class AsyncDatabaseRequest extends RichAsyncFunction> resultFuture) throws Exception { + public void asyncInvoke(String key, final ResultFuture> resultFuture) throws Exception { // issue the asynchronous request, receive a future for result - Future resultFuture = client.query(str); + final Future result = client.query(key); // set the callback to be executed once the request by the client is complete // the callback simply forwards the result to the result future - resultFuture.thenAccept( (String result) -> { - - resultFuture.complete(Collections.singleton(new Tuple2<>(str, result))); - + CompletableFuture.supplyAsync(new Supplier() { + + @Override + public String get() { + try { + return result.get(); + } catch (InterruptedException | ExecutionException e) { + // Normally handled explicitly. + return null; + } + } + }).thenAccept( (String dbResult) -> { + resultFuture.complete(Collections.singleton(new Tuple2<>(key, dbResult))); }); } } @@ -145,11 +154,11 @@ class AsyncDatabaseRequest extends AsyncFunction[String, (String, String)] { override def asyncInvoke(str: String, resultFuture: ResultFuture[(String, String)]): Unit = { // issue the asynchronous request, receive a future for the result - val resultFuture: Future[String] = client.query(str) + val resultFutureRequested: Future[String] = client.query(str) // set the callback to be executed once the request by the client is complete // the callback simply forwards the result to the result future - resultFuture.onSuccess { + resultFutureRequested.onSuccess { case result: String => resultFuture.complete(Iterable((str, result))) } } @@ -181,6 +190,12 @@ The following two parameters control the asynchronous operations: is exhausted. +### Timeout Handling + +When an async I/O request times out, by default an exception is thrown and job is restarted. +If you want to handle timeouts, you can override the `AsyncFunction#timeout` method. + + ### Order of Results The concurrent requests issued by the `AsyncFunction` frequently complete in some undefined order, based on which request finished first. diff --git a/docs/dev/stream/operators/index.md b/docs/dev/stream/operators/index.md index b0325ca46e3f7b..b89fa898fe7940 100644 --- a/docs/dev/stream/operators/index.md +++ b/docs/dev/stream/operators/index.md @@ -100,9 +100,9 @@ dataStream.filter(new FilterFunction() { KeyBy
    DataStream → KeyedStream -

    Logically partitions a stream into disjoint partitions, each partition containing elements of the same key. - Internally, this is implemented with hash partitioning. See keys on how to specify keys. - This transformation returns a KeyedStream.

    +

    Logically partitions a stream into disjoint partitions. All records with the same key are assigned to the same partition. Internally, keyBy() is implemented with hash partitioning. There are different ways to specify keys.

    +

    + This transformation returns a KeyedStream, which is, among other things, required to use keyed state.

    {% highlight java %} dataStream.keyBy("someKey") // Key by field "someKey" dataStream.keyBy(0) // Key by the first element of a Tuple @@ -123,9 +123,9 @@ dataStream.keyBy(0) // Key by the first element of a Tuple

    A "rolling" reduce on a keyed data stream. Combines the current element with the last reduced value and emits the new value. -
    +

    - A reduce function that creates a stream of partial sums:

    +

    A reduce function that creates a stream of partial sums:

    {% highlight java %} keyedStream.reduce(new ReduceFunction() { @Override @@ -306,6 +306,21 @@ dataStream.join(otherStream) .where().equalTo() .window(TumblingEventTimeWindows.of(Time.seconds(3))) .apply (new JoinFunction () {...}); + {% endhighlight %} + + + + Interval Join
    KeyedStream,KeyedStream → DataStream + +

    Join two elements e1 and e2 of two keyed streams with a common key over a given time interval, so that e1.timestamp + lowerBound <= e2.timestamp <= e1.timestamp + upperBound

    + {% highlight java %} +// this will join the two streams so that +// key1 == key2 && leftTs - 2 < rightTs < leftTs + 2 +keyedStream.intervalJoin(otherKeyedStream) + .between(Time.milliseconds(-2), Time.milliseconds(2)) // lower and upper bound + .upperBoundExclusive(true) // optional + .lowerBoundExclusive(true) // optional + .process(new IntervalJoinFunction() {...}); {% endhighlight %} @@ -419,14 +434,14 @@ IterativeStream iteration = initialStream.iterate(); DataStream iterationBody = iteration.map (/*do something*/); DataStream feedback = iterationBody.filter(new FilterFunction(){ @Override - public boolean filter(Integer value) throws Exception { + public boolean filter(Long value) throws Exception { return value > 0; } }); iteration.closeWith(feedback); DataStream output = iterationBody.filter(new FilterFunction(){ @Override - public boolean filter(Integer value) throws Exception { + public boolean filter(Long value) throws Exception { return value <= 0; } }); diff --git a/docs/dev/stream/operators/joining.md b/docs/dev/stream/operators/joining.md new file mode 100644 index 00000000000000..bdd8e427280262 --- /dev/null +++ b/docs/dev/stream/operators/joining.md @@ -0,0 +1,284 @@ +--- +title: "Joining" +nav-id: streaming_joins +nav-show_overview: true +nav-parent_id: streaming_operators +nav-pos: 11 +--- + + +* toc +{:toc} + +# Window Join +A window join joins the elements of two streams that share a common key and lie in the same window. These windows can be defined by using a [window assigner]({{ site.baseurl}}/dev/stream/operators/windows.html#window-assigners) and are evaluated on elements from both of the streams. + +The elements from both sides are then passed to a user-defined `JoinFunction` or `FlatJoinFunction` where the user can emit results that meet the join criteria. + +The general usage can be summarized as follows: + +{% highlight java %} +stream.join(otherStream) + .where() + .equalTo() + .window() + .apply() +{% endhighlight %} + +Some notes on semantics: +- The creation of pairwise combinations of elements of the two streams behaves like an inner-join, meaning elements from one stream will not be emitted if they don't have a corresponding element from the other stream to be joined with. +- Those elements that do get joined will have as their timestamp the largest timestamp that still lies in the respective window. For example a window with `[5, 10)` as its boundaries would result in the joined elements having 9 as their timestamp. + +In the following section we are going to give an overview over how different kinds of window joins behave using some exemplary scenarios. + +## Tumbling Window Join +When performing a tumbling window join, all elements with a common key and a common tumbling window are joined as pairwise combinations and passed on to a `JoinFunction` or `FlatJoinFunction`. Because this behaves like an inner join, elements of one stream that do not have elements from another stream in their tumbling window are not emitted! + + + +As illustrated in the figure, we define a tumbling window with the size of 2 milliseconds, which results in windows of the form `[0,1], [2,3], ...`. The image shows the pairwise combinations of all elements in each window which will be passed on to the `JoinFunction`. Note that in the tumbling window `[6,7]` nothing is emitted because no elements exist in the green stream to be joined with the orange elements ⑥ and ⑦. + +
    +
    +{% highlight java %} +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +DataStream orangeStream = ... +DataStream greenStream = ... + +orangeStream.join(greenStream) + .where() + .equalTo() + .window(TumblingEventTimeWindows.of(Time.milliseconds(2))) + .apply (new JoinFunction (){ + @Override + public String join(Integer first, Integer second) { + return first + "," + second; + } + }); +{% endhighlight %} +
    +
    + +{% highlight scala %} +import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +val orangeStream: DataStream[Integer] = ... +val greenStream: DataStream[Integer] = ... + +orangeStream.join(greenStream) + .where(elem => /* select key */) + .equalTo(elem => /* select key */) + .window(TumblingEventTimeWindows.of(Time.milliseconds(2))) + .apply { (e1, e2) => e1 + "," + e2 } +{% endhighlight %} + +
    +
    + +## Sliding Window Join +When performing a sliding window join, all elements with a common key and common sliding window are joined as pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Elements of one stream that do not have elements from the other stream in the current sliding window are not emitted! Note that some elements might be joined in one sliding window but not in another! + + + +In this example we are using sliding windows with a size of two milliseconds and slide them by one millisecond, resulting in the sliding windows `[-1, 0],[0,1],[1,2],[2,3], …`. The joined elements below the x-axis are the ones that are passed to the `JoinFunction` for each sliding window. Here you can also see how for example the orange ② is joined with the green ③ in the window `[2,3]`, but is not joined with anything in the window `[1,2]`. + +
    +
    + +{% highlight java %} +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +DataStream orangeStream = ... +DataStream greenStream = ... + +orangeStream.join(greenStream) + .where() + .equalTo() + .window(SlidingEventTimeWindows.of(Time.milliseconds(2) /* size */, Time.milliseconds(1) /* slide */)) + .apply (new JoinFunction (){ + @Override + public String join(Integer first, Integer second) { + return first + "," + second; + } + }); +{% endhighlight %} +
    +
    + +{% highlight scala %} +import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +val orangeStream: DataStream[Integer] = ... +val greenStream: DataStream[Integer] = ... + +orangeStream.join(greenStream) + .where(elem => /* select key */) + .equalTo(elem => /* select key */) + .window(SlidingEventTimeWindows.of(Time.milliseconds(2) /* size */, Time.milliseconds(1) /* slide */)) + .apply { (e1, e2) => e1 + "," + e2 } +{% endhighlight %} +
    +
    + +## Session Window Join +When performing a session window join, all elements with the same key that when _"combined"_ fulfill the session criteria are joined in pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Again this performs an inner join, so if there is a session window that only contains elements from one stream, no output will be emitted! + + + +Here we define a session window join where each session is divided by a gap of at least 1ms. There are three sessions, and in the first two sessions the joined elements from both streams are passed to the `JoinFunction`. In the third session there are no elements in the green stream, so ⑧ and ⑨ are not joined! + +
    +
    + +{% highlight java %} +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +DataStream orangeStream = ... +DataStream greenStream = ... + +orangeStream.join(greenStream) + .where() + .equalTo() + .window(EventTimeSessionWindows.withGap(Time.milliseconds(1))) + .apply (new JoinFunction (){ + @Override + public String join(Integer first, Integer second) { + return first + "," + second; + } + }); +{% endhighlight %} +
    +
    + +{% highlight scala %} +import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +val orangeStream: DataStream[Integer] = ... +val greenStream: DataStream[Integer] = ... + +orangeStream.join(greenStream) + .where(elem => /* select key */) + .equalTo(elem => /* select key */) + .window(EventTimeSessionWindows.withGap(Time.milliseconds(1))) + .apply { (e1, e2) => e1 + "," + e2 } +{% endhighlight %} + +
    +
    + +# Interval Join +The interval join joins elements of two streams (we'll call them A & B for now) with a common key and where elements of stream B have timestamps that lie in a relative time interval to timestamps of elements in stream A. + +This can also be expressed more formally as +`b.timestamp ∈ [a.timestamp + lowerBound; a.timestamp + upperBound]` or +`a.timestamp + lowerBound <= b.timestamp <= a.timestamp + upperBound` + +where a and b are elements of A and B that share a common key. Both the lower and upper bound can be either negative or positive as long as as the lower bound is always smaller or equal to the upper bound. The interval join currently only performs inner joins. + +When a pair of elements are passed to the `ProcessJoinFunction`, they will be assigned with the larger timestamp (which can be accessed via the `ProcessJoinFunction.Context`) of the two elements. + +Note The interval join currently only supports event time. + + + +In the example above, we join two streams 'orange' and 'green' with a lower bound of -2 milliseconds and an upper bound of +1 millisecond. Be default, these boundaries are inclusive, but `.lowerBoundExclusive()` and `.upperBoundExclusive` can be applied to change the behaviour. + +Using the more formal notation again this will translate to + +`orangeElem.ts + lowerBound <= greenElem.ts <= orangeElem.ts + upperBound` + +as indicated by the triangles. + +
    +
    + +{% highlight java %} +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +DataStream orangeStream = ... +DataStream greenStream = ... + +orangeStream + .keyBy() + .intervalJoin(greenStream.keyBy()) + .between(Time.milliseconds(-2), Time.milliseconds(1)) + .process (new ProcessJoinFunction out) { + out.collect(first + "," + second); + } + }); +{% endhighlight %} + +
    +
    + +{% highlight scala %} +import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction; +import org.apache.flink.streaming.api.windowing.time.Time; + +... + +val orangeStream: DataStream[Integer] = ... +val greenStream: DataStream[Integer] = ... + +orangeStream + .keyBy(elem => /* select key */) + .intervalJoin(greenStream.keyBy(elem => /* select key */)) + .between(Time.milliseconds(-2), Time.milliseconds(1)) + .process(new ProcessJoinFunction[Integer, Integer, String] { + override def processElement(left: Integer, right: Integer, ctx: ProcessJoinFunction[Integer, Integer, String]#Context, out: Collector[String]): Unit = { + out.collect(left + "," + right); + } + }); + }); +{% endhighlight %} + +
    +
    + +{% top %} diff --git a/docs/dev/stream/operators/process_function.md b/docs/dev/stream/operators/process_function.md index a52c5bfeb48580..f0189f9eeec4bd 100644 --- a/docs/dev/stream/operators/process_function.md +++ b/docs/dev/stream/operators/process_function.md @@ -242,4 +242,135 @@ class CountWithTimeoutFunction extends ProcessFunction[(String, String), (String the current processing time as event-time timestamp. This behavior is very subtle and might not be noticed by users. Well, it's harmful because processing-time timestamps are indeterministic and not aligned with watermarks. Besides, user-implemented logic depends on this wrong timestamp highly likely is unintendedly faulty. So we've decided to fix it. Upon upgrading to 1.4.0, Flink jobs -that are using this incorrect event-time timestamp will fail, and users should adapt their jobs to the correct logic. \ No newline at end of file +that are using this incorrect event-time timestamp will fail, and users should adapt their jobs to the correct logic. + +## The KeyedProcessFunction + +`KeyedProcessFunction`, as an extension of `ProcessFunction`, gives access to the key of timers in its `onTimer(...)` +method. + +
    +
    +{% highlight java %} +@Override +public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + K key = ctx.getCurrentKey(); + // ... +} + +{% endhighlight %} +
    + +
    +{% highlight scala %} +override def onTimer(timestamp: Long, ctx: OnTimerContext, out: Collector[OUT]): Unit = { + var key = ctx.getCurrentKey + // ... +} +{% endhighlight %} +
    +
    + +## Timers + +Both types of timers (processing-time and event-time) are internally maintained by the `TimerService` and enqueued for execution. + +The `TimerService` deduplicates timers per key and timestamp, i.e., there is at most one timer per key and timestamp. If multiple timers are registered for the same timestamp, the `onTimer()` method will be called just once. + +Note Flink synchronizes invocations of `onTimer()` and `processElement()`. Hence, users do not have to worry about concurrent modification of state. + +### Fault Tolerance + +Timers are fault tolerant and checkpointed along with the state of the application. +In case of a failure recovery or when starting an application from a savepoint, the timers are restored. + +Note Checkpointed processing-time timers that were supposed to fire before their restoration, will fire immediately. +This might happen when an application recovers from a failure or when it is started from a savepoint. + +Note Timers are always asynchronously checkpointed, except for the combination of RocksDB backend / with incremental snapshots / with heap-based timers (will be resolved with `FLINK-10026`). +Notice that large numbers of timers can increase the checkpointing time because timers are part of the checkpointed state. See the "Timer Coalescing" section for advice on how to reduce the number of timers. + +### Timer Coalescing + +Since Flink maintains only one timer per key and timestamp, you can reduce the number of timers by reducing the timer resolution to coalesce them. + +For a timer resolution of 1 second (event or processing time), you +can round down the target time to full seconds. Timers will fire at most 1 second earlier but not later than requested with millisecond accuracy. +As a result, there are at most one timer per key and second. + +
    +
    +{% highlight java %} +long coalescedTime = ((ctx.timestamp() + timeout) / 1000) * 1000; +ctx.timerService().registerProcessingTimeTimer(coalescedTime); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val coalescedTime = ((ctx.timestamp + timeout) / 1000) * 1000 +ctx.timerService.registerProcessingTimeTimer(coalescedTime) +{% endhighlight %} +
    +
    + +Since event-time timers only fire with watermarks coming in, you may also schedule and coalesce +these timers with the next watermark by using the current one: + +
    +
    +{% highlight java %} +long coalescedTime = ctx.timerService().currentWatermark() + 1; +ctx.timerService().registerEventTimeTimer(coalescedTime); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val coalescedTime = ctx.timerService.currentWatermark + 1 +ctx.timerService.registerEventTimeTimer(coalescedTime) +{% endhighlight %} +
    +
    + +Timers can also be stopped and removed as follows: + +Stopping a processing-time timer: + +
    +
    +{% highlight java %} +long timestampOfTimerToStop = ... +ctx.timerService().deleteProcessingTimeTimer(timestampOfTimerToStop); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val timestampOfTimerToStop = ... +ctx.timerService.deleteProcessingTimeTimer(timestampOfTimerToStop) +{% endhighlight %} +
    +
    + +Stopping an event-time timer: + +
    +
    +{% highlight java %} +long timestampOfTimerToStop = ... +ctx.timerService().deleteEventTimeTimer(timestampOfTimerToStop); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val timestampOfTimerToStop = ... +ctx.timerService.deleteEventTimeTimer(timestampOfTimerToStop) +{% endhighlight %} +
    +
    + +Note Stopping a timer has no effect if no such timer with the given timestamp is registered. + +{% top %} diff --git a/docs/dev/stream/operators/windows.md b/docs/dev/stream/operators/windows.md index 649bfe861c9462..5f6c8ac45b36b5 100644 --- a/docs/dev/stream/operators/windows.md +++ b/docs/dev/stream/operators/windows.md @@ -504,7 +504,7 @@ private static class AverageAggregate @Override public Double getResult(Tuple2 accumulator) { - return accumulator.f0 / accumulator.f1; + return ((double) accumulator.f0) / accumulator.f1; } @Override @@ -724,17 +724,19 @@ A `ProcessWindowFunction` can be defined and used like this: DataStream> input = ...; input - .keyBy() - .window() - .process(new MyProcessWindowFunction()); + .keyBy(t -> t.f0) + .timeWindow(Time.minutes(5)) + .process(new MyProcessWindowFunction()); /* ... */ -public class MyProcessWindowFunction implements ProcessWindowFunction, String, String, TimeWindow> { +public class MyProcessWindowFunction + extends ProcessWindowFunction, String, String, TimeWindow> { - void process(String key, Context context, Iterable> input, Collector out) { + @Override + public void process(String key, Context context, Iterable> input, Collector out) { long count = 0; - for (Tuple in: input) { + for (Tuple2 in: input) { count++; } out.collect("Window: " + context.window() + "count: " + count); @@ -749,9 +751,9 @@ public class MyProcessWindowFunction implements ProcessWindowFunction) - .window() - .process(new MyProcessWindowFunction()) + .keyBy(_._1) + .timeWindow(Time.minutes(5)) + .process(new MyProcessWindowFunction()) /* ... */ @@ -778,10 +780,10 @@ The example shows a `ProcessWindowFunction` that counts the elements in a window A `ProcessWindowFunction` can be combined with either a `ReduceFunction`, an `AggregateFunction`, or a `FoldFunction` to incrementally aggregate elements as they arrive in the window. When the window is closed, the `ProcessWindowFunction` will be provided with the aggregated result. -This allows to incrementally compute windows while having access to the +This allows it to incrementally compute windows while having access to the additional window meta information of the `ProcessWindowFunction`. -Note You can also the legacy `WindowFunction` instead of +Note You can also use the legacy `WindowFunction` instead of `ProcessWindowFunction` for incremental window aggregation. #### Incremental Window Aggregation with ReduceFunction @@ -797,7 +799,7 @@ DataStream input = ...; input .keyBy() - .timeWindow() + .timeWindow() .reduce(new MyReduceFunction(), new MyProcessWindowFunction()); // Function definitions @@ -810,14 +812,14 @@ private static class MyReduceFunction implements ReduceFunction { } private static class MyProcessWindowFunction - implements ProcessWindowFunction, String, TimeWindow> { + extends ProcessWindowFunction, String, TimeWindow> { public void process(String key, Context context, Iterable minReadings, Collector> out) { SensorReading min = minReadings.iterator().next(); - out.collect(new Tuple2(window.getStart(), min)); + out.collect(new Tuple2(context.window().getStart(), min)); } } @@ -830,16 +832,16 @@ val input: DataStream[SensorReading] = ... input .keyBy() - .timeWindow() + .timeWindow() .reduce( (r1: SensorReading, r2: SensorReading) => { if (r1.value > r2.value) r2 else r1 }, ( key: String, - window: TimeWindow, + context: ProcessWindowFunction[_, _, _, TimeWindow]#Context, minReadings: Iterable[SensorReading], out: Collector[(Long, SensorReading)] ) => { val min = minReadings.iterator.next() - out.collect((window.getStart, min)) + out.collect((context.window.getStart, min)) } ) @@ -856,11 +858,11 @@ the average.
    {% highlight java %} -DataStream input = ...; +DataStream> input = ...; input .keyBy() - .timeWindow() + .timeWindow() .aggregate(new AverageAggregate(), new MyProcessWindowFunction()); // Function definitions @@ -883,7 +885,7 @@ private static class AverageAggregate @Override public Double getResult(Tuple2 accumulator) { - return accumulator.f0 / accumulator.f1; + return ((double) accumulator.f0) / accumulator.f1; } @Override @@ -893,7 +895,7 @@ private static class AverageAggregate } private static class MyProcessWindowFunction - implements ProcessWindowFunction, String, TimeWindow> { + extends ProcessWindowFunction, String, TimeWindow> { public void process(String key, Context context, @@ -913,7 +915,7 @@ val input: DataStream[(String, Long)] = ... input .keyBy() - .timeWindow() + .timeWindow() .aggregate(new AverageAggregate(), new MyProcessWindowFunction()) // Function definitions @@ -959,7 +961,7 @@ DataStream input = ...; input .keyBy() - .timeWindow() + .timeWindow() .fold(new Tuple3("",0L, 0), new MyFoldFunction(), new MyProcessWindowFunction()) // Function definitions @@ -969,13 +971,13 @@ private static class MyFoldFunction public Tuple3 fold(Tuple3 acc, SensorReading s) { Integer cur = acc.getField(2); - acc.setField(2, cur + 1); + acc.setField(cur + 1, 2); return acc; } } private static class MyProcessWindowFunction - implements ProcessWindowFunction, Tuple3, String, TimeWindow> { + extends ProcessWindowFunction, Tuple3, String, TimeWindow> { public void process(String key, Context context, @@ -995,7 +997,7 @@ val input: DataStream[SensorReading] = ... input .keyBy() - .timeWindow() + .timeWindow() .fold ( ("", 0L, 0), (acc: (String, Long, Int), r: SensorReading) => { ("", 0L, acc._3 + 1) }, @@ -1032,7 +1034,7 @@ different keys and events for all of them currently fall into the *[12:00, 13:00 then there will be 1000 window instances that each have their own keyed per-window state. There are two methods on the `Context` object that a `process()` invocation receives that allow -access two the two types of state: +access to the two types of state: - `globalState()`, which allows access to keyed state that is not scoped to a window - `windowState()`, which allows access to keyed state that is also scoped to the window @@ -1412,7 +1414,6 @@ val globalResults = resultsPerKey In this example, the results for time window `[0, 5)` from the first operation will also end up in time window `[0, 5)` in the subsequent windowed operation. This allows calculating a sum per key and then calculating the top-k elements within the same window in the second operation. -and then calculating the top-k elements within the same window in the second operation. ## Useful state size considerations diff --git a/docs/dev/stream/python.md b/docs/dev/stream/python.md index a533819f5026d2..c8d59e31b728c5 100644 --- a/docs/dev/stream/python.md +++ b/docs/dev/stream/python.md @@ -24,6 +24,10 @@ specific language governing permissions and limitations under the License. --> +Attention *This API is based on Jython, +which is not a full Python replacement and may restrict the libraries you are able +to use with your application (see below for more information).* + Analysis streaming programs in Flink are regular programs that implement transformations on streaming data sets (e.g., filtering, mapping, joining, grouping). The streaming data sets are initially created from certain sources (e.g., by reading files, or from collections). @@ -50,7 +54,12 @@ existing Java streaming APIs. There are two main constraints for using Jython: * The latest Python supported version is 2.7 -* It is not straightforward to use Python C extensions +* It is not straightforward to use Python C extensions, which may prevent use of some libraries + +(For more information please see .) + +One possible alternative for streaming that allows for native Python execution would be the [Apache Beam +portability framework](https://beam.apache.org/contribute/portability/) with the Flink runner. Streaming Program Example ------------------------- @@ -104,7 +113,7 @@ def main(factory): .key_by(Selector()) \ .time_window(milliseconds(50)) \ .reduce(Sum()) \ - .print() + .output() env.execute() {% endhighlight %} @@ -227,7 +236,7 @@ Data transformations transform one or more DataStreams into a new DataStream. Pr multiple transformations into sophisticated assemblies. This section gives a brief overview of the available transformations. The [transformations -documentation](dataset_transformations.html) has a full description of all transformations with +documentation](./operators/index.html) has a full description of all transformations with examples.
    @@ -322,7 +331,7 @@ data.reduce(Sum())

    Windows can be defined on already partitioned KeyedStreams. Windows group the data in each key according to some characteristic (e.g., the data that arrived within the last 5 seconds). - See windows for a complete description of windows. + See windows for a complete description of windows. {% highlight python %} keyed_stream.count_window(10, 5) # Last 10 elements, sliding (jumping) by 5 elements @@ -464,7 +473,7 @@ Rich functions (.e.g `RichFilterFunction`) enable to define (override) the optio The user may use these functions for initialization and cleanups. {% highlight python %} -class Tockenizer(RichMapFunction): +class Tokenizer(RichMapFunction): def open(self, config): pass def close(self): @@ -472,7 +481,7 @@ class Tockenizer(RichMapFunction): def map(self, value): pass -data_stream.map(Tockenizer()) +data_stream.map(Tokenizer()) {% endhighlight %} The `open` function is called by the worker before starting the streaming pipeline. @@ -624,7 +633,7 @@ env.execute() A system-wide default parallelism for all execution environments can be defined by setting the `parallelism.default` property in `./conf/flink-conf.yaml`. See the -[Configuration]({{ site.baseurl }}/setup/config.html) documentation for details. +[Configuration]({{ site.baseurl }}/ops/config.html) documentation for details. {% top %} diff --git a/docs/dev/stream/side_output.md b/docs/dev/stream/side_output.md index c694e47534d01b..7ca195d7022472 100644 --- a/docs/dev/stream/side_output.md +++ b/docs/dev/stream/side_output.md @@ -58,8 +58,9 @@ contains. Emitting data to a side output is possible from the following functions: - [ProcessFunction]({{ site.baseurl }}/dev/stream/operators/process_function.html) +- [KeyedProcessFunction]({{ site.baseurl }}/dev/stream/operators/process_function.html#the-keyedprocessfunction) - CoProcessFunction -- [ProcessWindowFunction]({{ site.baseurl }}/dev/windows.html#processwindowfunction) +- [ProcessWindowFunction]({{ site.baseurl }}/dev/stream/operators/windows.html#processwindowfunction) - ProcessAllWindowFunction You can use the `Context` parameter, which is exposed to users in the above functions, to emit diff --git a/docs/dev/stream/state/broadcast_state.md b/docs/dev/stream/state/broadcast_state.md new file mode 100644 index 00000000000000..73f4dcf7fe4e03 --- /dev/null +++ b/docs/dev/stream/state/broadcast_state.md @@ -0,0 +1,279 @@ +--- +title: "The Broadcast State Pattern" +nav-parent_id: streaming_state +nav-pos: 2 +--- + + +* ToC +{:toc} + +[Working with State](state.html) describes operator state which upon restore is either evenly distributed among the +parallel tasks of an operator, or unioned, with the whole state being used to initialize the restored parallel tasks. + +A third type of supported *operator state* is the *Broadcast State*. Broadcast state was introduced to support use cases +where some data coming from one stream is required to be broadcasted to all downstream tasks, where it is stored locally +and is used to process all incoming elements on the other stream. As an example where broadcast state can emerge as a +natural fit, one can imagine a low-throughput stream containing a set of rules which we want to evaluate against all +elements coming from another stream. Having the above type of use cases in mind, broadcast state differs from the rest +of operator states in that: + 1. it has a map format, + 2. it is only available to specific operators that have as inputs a *broadcasted* stream and a *non-broadcasted* one, and + 3. such an operator can have *multiple broadcast states* with different names. + +## Provided APIs + +To show the provided APIs, we will start with an example before presenting their full functionality. As our running +example, we will use the case where we have a stream of objects of different colors and shapes and we want to find pairs +of objects of the same color that follow a certain pattern, *e.g.* a rectangle followed by a triangle. We assume that +the set of interesting patterns evolves over time. + +In this example, the first stream will contain elements of type `Item` with a `Color` and a `Shape` property. The other +stream will contain the `Rules`. + +Starting from the stream of `Items`, we just need to *key it* by `Color`, as we want pairs of the same color. This will +make sure that elements of the same color end up on the same physical machine. + +{% highlight java %} +// key the shapes by color +KeyedStream colorPartitionedStream = shapeStream + .keyBy(new KeySelector(){...}); +{% endhighlight %} + +Moving on to the `Rules`, the stream containing them should be broadcasted to all downstream tasks, and these tasks +should store them locally so that they can evaluate them against all incoming `Items`. The snippet below will i) broadcast +the stream of rules and ii) using the provided `MapStateDescriptor`, it will create the broadcast state where the rules +will be stored. + +{% highlight java %} + +// a map descriptor to store the name of the rule (string) and the rule itself. +MapStateDescriptor ruleStateDescriptor = new MapStateDescriptor<>( + "RulesBroadcastState", + BasicTypeInfo.STRING_TYPE_INFO, + TypeInformation.of(new TypeHint() {})); + +// broadcast the rules and create the broadcast state +BroadcastStream ruleBroadcastStream = ruleStream + .broadcast(ruleStateDescriptor); +{% endhighlight %} + +Finally, in order to evaluate the `Rules` against the incoming elements from the `Item` stream, we need to: + 1. connect the two streams, and + 2. specify our match detecting logic. + +Connecting a stream (keyed or non-keyed) with a `BroadcastStream` can be done by calling `connect()` on the +non-broadcasted stream, with the `BroadcastStream` as an argument. This will return a `BroadcastConnectedStream`, on +which we can call `process()` with a special type of `CoProcessFunction`. The function will contain our matching logic. +The exact type of the function depends on the type of the non-broadcasted stream: + - if that is **keyed**, then the function is a `KeyedBroadcastProcessFunction`. + - if it is **non-keyed**, the function is a `BroadcastProcessFunction`. + + Given that our non-broadcasted stream is keyed, the following snippet includes the above calls: + +

    + Attention: The connect should be called on the non-broadcasted stream, with the BroadcastStream + as an argument. +
    + +{% highlight java %} +DataStream output = colorPartitionedStream + .connect(ruleBroadcastStream) + .process( + + // type arguments in our KeyedBroadcastProcessFunction represent: + // 1. the key of the keyed stream + // 2. the type of elements in the non-broadcast side + // 3. the type of elements in the broadcast side + // 4. the type of the result, here a string + + new KeyedBroadcastProcessFunction() { + // my matching logic + } + ); +{% endhighlight %} + +### BroadcastProcessFunction and KeyedBroadcastProcessFunction + +As in the case of a `CoProcessFunction`, these functions have two process methods to implement; the `processBroadcastElement()` +which is responsible for processing incoming elements in the broadcasted stream and the `processElement()` which is used +for the non-broadcasted one. The full signatures of the methods are presented below: + +{% highlight java %} +public abstract class BroadcastProcessFunction extends BaseBroadcastProcessFunction { + + public abstract void processElement(IN1 value, ReadOnlyContext ctx, Collector out) throws Exception; + + public abstract void processBroadcastElement(IN2 value, Context ctx, Collector out) throws Exception; +} +{% endhighlight %} + +{% highlight java %} +public abstract class KeyedBroadcastProcessFunction { + + public abstract void processElement(IN1 value, ReadOnlyContext ctx, Collector out) throws Exception; + + public abstract void processBroadcastElement(IN2 value, Context ctx, Collector out) throws Exception; + + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception; +} +{% endhighlight %} + +The first thing to notice is that both functions require the implementation of the `processBroadcastElement()` method +for processing elements in the broadcast side and the `processElement()` for elements in the non-broadcasted side. + +The two methods differ in the context they are provided. The non-broadcast side has a `ReadOnlyContext`, while the +broadcasted side has a `Context`. + +Both of these contexts (`ctx` in the following enumeration): + 1. give access to the broadcast state: `ctx.getBroadcastState(MapStateDescriptor stateDescriptor)` + 2. allow to query the timestamp of the element: `ctx.timestamp()`, + 3. get the current watermark: `ctx.currentWatermark()` + 4. get the current processing time: `ctx.currentProcessingTime()`, and + 5. emit elements to side-outputs: `ctx.output(OutputTag outputTag, X value)`. + +The `stateDescriptor` in the `getBroadcastState()` should be identical to the one in the `.broadcast(ruleStateDescriptor)` +above. + +The difference lies in the type of access each one gives to the broadcast state. The broadcasted side has +**read-write access** to it, while the non-broadcast side has **read-only access** (thus the names). The reason for this +is that in Flink there is no cross-task communication. So, to guarantee that the contents in the Broadcast State are the +same across all parallel instances of our operator, we give read-write access only to the broadcast side, which sees the +same elements across all tasks, and we require the computation on each incoming element on that side to be identical +across all tasks. Ignoring this rule would break the consistency guarantees of the state, leading to inconsistent and +often difficult to debug results. + +
    + Attention: The logic implemented in `processBroadcast()` must have the same deterministic behavior + across all parallel instances! +
    + +Finally, due to the fact that the `KeyedBroadcastProcessFunction` is operating on a keyed stream, it +exposes some functionality which is not available to the `BroadcastProcessFunction`. That is: + 1. the `ReadOnlyContext` in the `processElement()` method gives access to Flink's underlying timer service, which allows + to register event and/or processing time timers. When a timer fires, the `onTimer()` (shown above) is invoked with an + `OnTimerContext` which exposes the same functionality as the `ReadOnlyContext` plus + - the ability to ask if the timer that fired was an event or processing time one and + - to query the key associated with the timer. + 2. the `Context` in the `processBroadcastElement()` method contains the method + `applyToKeyedState(StateDescriptor stateDescriptor, KeyedStateFunction function)`. This allows to + register a `KeyedStateFunction` to be **applied to all states of all keys** associated with the provided `stateDescriptor`. + +
    + Attention: Registering timers is only possible at `processElement()` of the `KeyedBroadcastProcessFunction` + and only there. It is not possible in the `processBroadcastElement()` method, as there is no key associated to the + broadcasted elements. +
    + +Coming back to our original example, our `KeyedBroadcastProcessFunction` could look like the following: + +{% highlight java %} +new KeyedBroadcastProcessFunction() { + + // store partial matches, i.e. first elements of the pair waiting for their second element + // we keep a list as we may have many first elements waiting + private final MapStateDescriptor> mapStateDesc = + new MapStateDescriptor<>( + "items", + BasicTypeInfo.STRING_TYPE_INFO, + new ListTypeInfo<>(Item.class)); + + // identical to our ruleStateDescriptor above + private final MapStateDescriptor ruleStateDescriptor = + new MapStateDescriptor<>( + "RulesBroadcastState", + BasicTypeInfo.STRING_TYPE_INFO, + TypeInformation.of(new TypeHint() {})); + + @Override + public void processBroadcastElement(Rule value, + Context ctx, + Collector out) throws Exception { + ctx.getBroadcastState(ruleStateDescriptor).put(value.name, value); + } + + @Override + public void processElement(Item value, + ReadOnlyContext ctx, + Collector out) throws Exception { + + final MapState> state = getRuntimeContext().getMapState(mapStateDesc); + final Shape shape = value.getShape(); + + for (Map.Entry entry : + ctx.getBroadcastState(ruleStateDescriptor).immutableEntries()) { + final String ruleName = entry.getKey(); + final Rule rule = entry.getValue(); + + List stored = state.get(ruleName); + if (stored == null) { + stored = new ArrayList<>(); + } + + if (shape == rule.second && !stored.isEmpty()) { + for (Item i : stored) { + out.collect("MATCH: " + i + " - " + value); + } + stored.clear(); + } + + // there is no else{} to cover if rule.first == rule.second + if (shape.equals(rule.first)) { + stored.add(value); + } + + if (stored.isEmpty()) { + state.remove(ruleName); + } else { + state.put(ruleName, stored); + } + } + } +} +{% endhighlight %} + +## Important Considerations + +After describing the offered APIs, this section focuses on the important things to keep in mind when using broadcast +state. These are: + + - **There is no cross-task communication:** As stated earlier, this is the reason why only the broadcast side of a +`(Keyed)-BroadcastProcessFunction` can modify the contents of the broadcast state. In addition, the user has to make +sure that all tasks modify the contents of the broadcast state in the same way for each incoming element. Otherwise, +different tasks might have different contents, leading to inconsistent results. + + - **Order of events in Broadcast State may differ across tasks:** Although broadcasting the elements of a stream +guarantees that all elements will (eventually) go to all downstream tasks, elements may arrive in a different order +to each task. So the state updates for each incoming element *MUST NOT depend on the ordering* of the incoming +events. + + - **All tasks checkpoint their broadcast state:** Although all tasks have the same elements in their broadcast state +when a checkpoint takes place (checkpoint barriers do not overpass elements), all tasks checkpoint their broadcast state, +and not just one of them. This is a design decision to avoid having all tasks read from the same file during a restore +(thus avoiding hotspots), although it comes at the expense of increasing the size of the checkpointed state by a factor +of p (= parallelism). Flink guarantees that upon restoring/rescaling there will be **no duplicates** and **no missing data**. +In case of recovery with the same or smaller parallelism, each task reads its checkpointed state. Upon scaling up, each +task reads its own state, and the remaining tasks (`p_new`-`p_old`) read checkpoints of previous tasks in a round-robin +manner. + + - **No RocksDB state backend:** Broadcast state is kept in-memory at runtime and memory provisioning should be done +accordingly. This holds for all operator states. + +{% top %} diff --git a/docs/dev/stream/state/checkpointing.md b/docs/dev/stream/state/checkpointing.md index f856d7d2351ea2..c8c17f1f2b9931 100644 --- a/docs/dev/stream/state/checkpointing.md +++ b/docs/dev/stream/state/checkpointing.md @@ -1,7 +1,7 @@ --- title: "Checkpointing" nav-parent_id: streaming_state -nav-pos: 2 +nav-pos: 3 --- -If your application uses Flink's managed state, it might be necessary to implement custom serialization logic for special use cases. +* ToC +{:toc} -This page is targeted as a guideline for users who require the use of custom serialization for their state, covering how -to provide a custom serializer and how to handle upgrades to the serializer for compatibility. If you're simply using -Flink's own serializers, this page is irrelevant and can be skipped. +This page is targeted as a guideline for users who require the use of custom serialization for their state, covering +how to provide a custom state serializer as well as guidelines and best practices for implementing serializers that allow +state schema evolution. -### Using custom serializers +If you're simply using Flink's own serializers, this page is irrelevant and can be ignored. -As demonstrated in the above examples, when registering a managed operator or keyed state, a `StateDescriptor` is required +## Using custom state serializers + +When registering a managed operator or keyed state, a `StateDescriptor` is required to specify the state's name, as well as information about the type of the state. The type information is used by Flink's [type serialization framework](../../types_serialization.html) to create appropriate serializers for the state. @@ -66,125 +69,204 @@ checkpointedState = getRuntimeContext.getListState(descriptor)
    -Note that Flink writes state serializers along with the state as metadata. In certain cases on restore (see following -subsections), the written serializer needs to be deserialized and used. Therefore, it is recommended to avoid using -anonymous classes as your state serializers. Anonymous classes do not have a guarantee on the generated classname, -which varies across compilers and depends on the order that they are instantiated within the enclosing class, which can -easily cause the previously written serializer to be unreadable (since the original class can no longer be found in the -classpath). - -### Handling serializer upgrades and compatibility - -Flink allows changing the serializers used to read and write managed state, so that users are not locked in to any -specific serialization. When state is restored, the new serializer registered for the state (i.e., the serializer -that comes with the `StateDescriptor` used to access the state in the restored job) will be checked for compatibility, -and is replaced as the new serializer for the state. - -A compatible serializer would mean that the serializer is capable of reading previous serialized bytes of the state, -and the written binary format of the state also remains identical. The means to check the new serializer's compatibility -is provided through the following two methods of the `TypeSerializer` interface: - -{% highlight java %} -public abstract TypeSerializerConfigSnapshot snapshotConfiguration(); -public abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); -{% endhighlight %} - -Briefly speaking, every time a checkpoint is performed, the `snapshotConfiguration` method is called to create a -point-in-time view of the state serializer's configuration. The returned configuration snapshot is stored along with the -checkpoint as the state's metadata. When the checkpoint is used to restore a job, that serializer configuration snapshot -will be provided to the _new_ serializer of the same state via the counterpart method, `ensureCompatibility`, to verify -compatibility of the new serializer. This method serves as a check for whether or not the new serializer is compatible, -as well as a hook to possibly reconfigure the new serializer in the case that it is incompatible. +## State serializers and schema evolution -Note that Flink's own serializers are implemented such that they are at least compatible with themselves, i.e. when the -same serializer is used for the state in the restored job, the serializer's will reconfigure themselves to be compatible -with their previous configuration. +This section explains the user-facing abstractions related to state serialization and schema evolution, and necessary +internal details about how Flink interacts with these abstractions. -The following subsections illustrate guidelines to implement these two methods when using custom serializers. +When restoring from savepoints, Flink allows changing the serializers used to read and write previously registered state, +so that users are not locked in to any specific serialization schema. When state is restored, a new serializer will be +registered for the state (i.e., the serializer that comes with the `StateDescriptor` used to access the state in the +restored job). This new serializer may have a different schema than that of the previous serializer. Therefore, when +implementing state serializers, besides the basic logic of reading / writing data, another important thing to keep in +mind is how the serialization schema can be changed in the future. -#### Implementing the `snapshotConfiguration` method +When speaking of *schema*, in this context the term is interchangeable between referring to the *data model* of a state +type and the *serialized binary format* of a state type. The schema, generally speaking, can change for a few cases: -The serializer's configuration snapshot should capture enough information such that on restore, the information -carried over to the new serializer for the state is sufficient for it to determine whether or not it is compatible. -This could typically contain information about the serializer's parameters or binary format of the serialized data; -generally, anything that allows the new serializer to decide whether or not it can be used to read previous serialized -bytes, and that it writes in the same binary format. + 1. Data schema of the state type has evolved, i.e. adding or removing a field from a POJO that is used as state. + 2. Generally speaking, after a change to the data schema, the serialization format of the serializer will need to be upgraded. + 3. Configuration of the serializer has changed. + +In order for the new execution to have information about the *written schema* of state and detect whether or not the +schema has changed, upon taking a savepoint of an operator's state, a *snapshot* of the state serializer needs to be +written along with the state bytes. This is abstracted a `TypeSerializerSnapshot`, explained in the next subsection. -How the serializer's configuration snapshot is written to and read from checkpoints is fully customizable. The below -is the base class for all serializer configuration snapshot implementations, the `TypeSerializerConfigSnapshot`. +### The `TypeSerializerSnapshot` abstraction +
    {% highlight java %} -public abstract TypeSerializerConfigSnapshot extends VersionedIOReadableWritable { - public abstract int getVersion(); - public void read(DataInputView in) {...} - public void write(DataOutputView out) {...} +public interface TypeSerializerSnapshot { + int getCurrentVersion(); + void writeSnapshot(DataOuputView out) throws IOException; + void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException; + TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer); + TypeSerializer restoreSerializer(); } {% endhighlight %} +
    -The `read` and `write` methods define how the configuration is read from and written to the checkpoint. The base -implementations contain logic to read and write the version of the configuration snapshot, so it should be extended and -not completely overridden. - -The version of the configuration snapshot is determined through the `getVersion` method. Versioning for the serializer -configuration snapshot is the means to maintain compatible configurations, as information included in the configuration -may change over time. By default, configuration snapshots are only compatible with the current version (as returned by -`getVersion`). To indicate that the configuration is compatible with other versions, override the `getCompatibleVersions` -method to return more version values. When reading from the checkpoint, you can use the `getReadVersion` method to -determine the version of the written configuration and adapt the read logic to the specific version. - -Attention The version of the serializer's configuration snapshot is **not** -related to upgrading the serializer. The exact same serializer can have different implementations of its -configuration snapshot, for example when more information is added to the configuration to allow more comprehensive -compatibility checks in the future. - -One limitation of implementing a `TypeSerializerConfigSnapshot` is that an empty constructor must be present. The empty -constructor is required when reading the configuration snapshot from checkpoints. - -#### Implementing the `ensureCompatibility` method - -The `ensureCompatibility` method should contain logic that performs checks against the information about the previous -serializer carried over via the provided `TypeSerializerConfigSnapshot`, basically doing one of the following: - - * Check whether the serializer is compatible, while possibly reconfiguring itself (if required) so that it may be - compatible. Afterwards, acknowledge with Flink that the serializer is compatible. - - * Acknowledge that the serializer is incompatible and that state migration is required before Flink can proceed with - using the new serializer. - -The above cases can be translated to code by returning one of the following from the `ensureCompatibility` method: - - * **`CompatibilityResult.compatible()`**: This acknowledges that the new serializer is compatible, or has been reconfigured to - be compatible, and Flink can proceed with the job with the serializer as is. - - * **`CompatibilityResult.requiresMigration()`**: This acknowledges that the serializer is incompatible, or cannot be - reconfigured to be compatible, and requires a state migration before the new serializer can be used. State migration - is performed by using the previous serializer to read the restored state bytes to objects, and then serialized again - using the new serializer. - - * **`CompatibilityResult.requiresMigration(TypeDeserializer deserializer)`**: This acknowledgement has equivalent semantics - to `CompatibilityResult.requiresMigration()`, but in the case that the previous serializer cannot be found or loaded - to read the restored state bytes for the migration, a provided `TypeDeserializer` can be used as a fallback resort. - -Attention Currently, as of Flink 1.3, if the result of the compatibility check -acknowledges that state migration needs to be performed, the job simply fails to restore from the checkpoint as state -migration is currently not available. The ability to migrate state will be introduced in future releases. - -### Managing `TypeSerializer` and `TypeSerializerConfigSnapshot` classes in user code - -Since `TypeSerializer`s and `TypeSerializerConfigSnapshot`s are written as part of checkpoints along with the state -values, the availability of the classes within the classpath may affect restore behaviour. - -`TypeSerializer`s are directly written into checkpoints using Java Object Serialization. In the case that the new -serializer acknowledges that it is incompatible and requires state migration, it will be required to be present to be -able to read the restored state bytes. Therefore, if the original serializer class no longer exists or has been modified -(resulting in a different `serialVersionUID`) as a result of a serializer upgrade for the state, the restore would -not be able to proceed. The alternative to this requirement is to provide a fallback `TypeDeserializer` when -acknowledging that state migration is required, using `CompatibilityResult.requiresMigration(TypeDeserializer deserializer)`. +
    +{% highlight java %} +public abstract class TypeSerializer { + + // ... + + public abstract TypeSerializerSnapshot snapshotConfiguration(); +} +{% endhighlight %} +
    -The class of `TypeSerializerConfigSnapshot`s in the restored checkpoint must exist in the classpath, as they are -fundamental components to compatibility checks on upgraded serializers and would not be able to be restored if the class -is not present. Since configuration snapshots are written to checkpoints using custom serialization, the implementation -of the class is free to be changed, as long as compatibility of the configuration change is handled using the versioning -mechanisms in `TypeSerializerConfigSnapshot`. +A serializer's `TypeSerializerSnapshot` is a point-in-time information that serves as the single source of truth about +the state serializer's write schema, as well as any additional information mandatory to restore a serializer that +would be identical to the given point-in-time. The logic about what should be written and read at restore time +as the serializer snapshot is defined in the `writeSnapshot` and `readSnapshot` methods. + +Note that the snapshot's own write schema may also need to change over time (e.g. when you wish to add more information +about the serializer to the snapshot). To facilitate this, snapshots are versioned, with the current version +number defined in the `getCurrentVersion` method. On restore, when the serializer snapshot is read from savepoints, +the version of the schema in which the snapshot was written in will be provided to the `readSnapshot` method so that +the read implementation can handle different versions. + +At restore time, the logic that detects whether or not the new serializer's schema has changed should be implemented in +the `resolveSchemaCompatibility` method. When previous registered state is registered again with new serializers in the +restored execution of an operator, the new serializer is provided to the previous serializer's snapshot via this method. +This method returns a `TypeSerializerSchemaCompatibility` representing the result of the compatibility resolution, +which can be one of the following: + + 1. **`TypeSerializerSchemaCompatibility.compatibleAsIs()`**: this result signals that the new serializer is compatible, + meaning that the new serializer has identical schema with the previous serializer. It is possible that the new + serializer has been reconfigured in the `resolveSchemaCompatibility` method so that it is compatible. + 2. **`TypeSerializerSchemaCompatibility.compatibleAfterMigration()`**: this result signals that the new serializer has a + different serialization schema, and it is possible to migrate from the old schema by using the previous serializer + (which recognizes the old schema) to read bytes into state objects, and then rewriting the object back to bytes with + the new serializer (which recognizes the new schema). + 3. **`TypeSerializerSchemaCompatibility.incompatible()`**: this result signals that the new serializer has a + different serialization schema, but it is not possible to migrate from the old schema. + +The last bit of detail is how the previous serializer is obtained in the case that migration is required. +Another important role of a serializer's `TypeSerializerSnapshot` is that it serves as a factory to restore +the previous serializer. More specifically, the `TypeSerializerSnapshot` should implement the `restoreSerializer` method +to instantiate a serializer instance that recognizes the previous serializer's schema and configuration, and can therefore +safely read data written by the previous serializer. + +### How Flink interacts with the `TypeSerializer` and `TypeSerializerSnapshot` abstractions + +To wrap up, this section concludes how Flink, or more specifically the state backends, interact with the +abstractions. The interaction is slightly different depending on the state backend, but this is orthogonal +to the implementation of state serializers and their serializer snapshots. + +#### Off-heap state backends (e.g. `RocksDBStateBackend`) + + 1. **Register new state with a state serializer that has schema _A_** + - the registered `TypeSerializer` for the state is used to read / write state on every state access. + - State is written in schema *A*. + 2. **Take a savepoint** + - The serializer snapshot is extracted via the `TypeSerializer#snapshotConfiguration` method. + - The serializer snapshot is written to the savepoint, as well as the already-serialized state bytes (with schema *A*). + 3. **Restored execution re-accesses restored state bytes with new state serializer that has schema _B_** + - The previous state serializer's snapshot is restored. + - State bytes are not deserialized on restore, only loaded back to the state backends (therefore, still in schema *A*). + - Upon receiving the new serializer, it is provided to the restored previous serializer's snapshot via the + `TypeSerializer#resolveSchemaCompatibility` to check for schema compatibility. + 4. **Migrate state bytes in backend from schema _A_ to schema _B_** + - If the compatibility resolution reflects that the schema has changed and migration is possible, schema migration is + performed. The previous state serializer which recognizes schema _A_ will be obtained from the serializer snapshot, via + `TypeSerializerSnapshot#restoreSerializer()`, and is used to deserialize state bytes to objects, which in turn + are re-written again with the new serializer, which recognizes schema _B_ to complete the migration. All entries + of the accessed state is migrated all-together before processing continues. + - If the resolution signals incompatibility, then the state access fails with an exception. + +#### Heap state backends (e.g. `MemoryStateBackend`, `FsStateBackend`) + + 1. **Register new state with a state serializer that has schema _A_** + - the registered `TypeSerializer` is maintained by the state backend. + 2. **Take a savepoint, serializing all state with schema _A_** + - The serializer snapshot is extracted via the `TypeSerializer#snapshotConfiguration` method. + - The serializer snapshot is written to the savepoint. + - State objects are now serialized to the savepoint, written in schema _A_. + 3. **On restore, deserialize state into objects in heap** + - The previous state serializer's snapshot is restored. + - The previous serializer, which recognizes schema _A_, is obtained from the serializer snapshot, via + `TypeSerializerSnapshot#restoreSerializer()`, and is used to deserialize state bytes to objects. + - From now on, all of the state is already deserialized. + 4. **Restored execution re-accesses previous state with new state serializer that has schema _B_** + - Upon receiving the new serializer, it is provided to the restored previous serializer's snapshot via the + `TypeSerializer#resolveSchemaCompatibility` to check for schema compatibility. + - If the compatibility check signals that migration is required, nothing happens in this case since for + heap backends, all state is already deserialized into objects. + - If the resolution signals incompatibility, then the state access fails with an exception. + 5. **Take another savepoint, serializing all state with schema _B_** + - Same as step 2., but now state bytes are all in schema _B_. + +## Implementation notes and best practices + +#### 1. Flink restores serializer snapshots by instantiating them with their classname + +A serializer's snapshot, being the single source of truth for how a registered state was serialized, serves as an +entry point to reading state in savepoints. In order to be able to restore and access previous state, the previous state +serializer's snapshot must be able to be restored. + +Flink restores serializer snapshots by first instantiating the `TypeSerializerSnapshot` with its classname (written +along with the snapshot bytes). Therefore, to avoid being subject to unintended classname changes or instantiation +failures, `TypeSerializerSnapshot` classes should: + + - avoid being implemented as anonymous classes or nested classes, + - have a public, nullary constructor for instantiation + +#### 2. Avoid sharing the same `TypeSerializerSnapshot` class across different serializers + +Since schema compatibility checks goes through the serializer snapshots, having multiple serializers returning +the same `TypeSerializerSnapshot` class as their snapshot would complicate the implementation for the +`TypeSerializerSnapshot#resolveSchemaCompatibility` and `TypeSerializerSnapshot#restoreSerializer()` method. + +This would also be a bad separation of concerns; a single serializer's serialization schema, +configuration, as well as how to restore it, should be consolidated in its own dedicated `TypeSerializerSnapshot` class. + +#### 3. Use the `CompositeSerializerSnapshot` utility for serializers that contain nested serializers + +There may be cases where a `TypeSerializer` relies on other nested `TypeSerializer`s; take for example Flink's +`TupleSerializer`, where it is configured with nested `TypeSerializer`s for the tuple fields. In this case, +the snapshot of the most outer serializer should also contain snapshots of the nested serializers. + +The `CompositeSerializerSnapshot` can be used specifically for this scenario. It wraps the logic of resolving +the overall schema compatibility check result for the composite serializer. +For an example of how it should be used, one can refer to Flink's +[ListSerializerSnapshot](https://github.com/apache/flink/blob/master/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java) implementation. + +## Migrating from deprecated serializer snapshot APIs before Flink 1.7 + +This section is a guide for API migration from serializers and serializer snapshots that existed before Flink 1.7. + +Before Flink 1.7, serializer snapshots were implemented as a `TypeSerializerConfigSnapshot` (which is now deprecated, +and will eventually be removed in the future to be fully replaced by the new `TypeSerializerSnapshot` interface). +Moreover, the responsibility of serializer schema compatibility checks lived within the `TypeSerializer`, +implemented in the `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` method. + +Another major difference between the new and old abstractions is that the deprecated `TypeSerializerConfigSnapshot` +did not have the capability of instantiating the previous serializer. Therefore, in the case where your serializer +still returns a subclass of `TypeSerializerConfigSnapshot` as its snapshot, the serializer instance itself will always +be written to savepoints using Java serialization so that the previous serializer may be available at restore time. +This is very undesirable, since whether or not restoring the job will be successful is susceptible to availability +of the previous serializer's class, or in general, whether or not the serializer instance can be read back at restore +time using Java serialization. This means that you be limited to the same serializer for your state, +and could be problematic once you want to upgrade serializer classes or perform schema migration. + +To be future-proof and have flexibility to migrate your state serializers and schema, it is highly recommended to +migrate from the old abstractions. The steps to do this is as follows: + + 1. Implement a new subclass of `TypeSerializerSnapshot`. This will be the new snapshot for your serializer. + 2. Return the new `TypeSerializerSnapshot` as the serializer snapshot for your serializer in the + `TypeSerializer#snapshotConfiguration()` method. + 3. Restore the job from the savepoint that existed before Flink 1.7, and then take a savepoint again. + Note that at this step, the old `TypeSerializerConfigSnapshot` of the serializer must still exist in the classpath, + and the implementation for the `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` method must not be + removed. The purpose of this process is to replace the `TypeSerializerConfigSnapshot` written in old savepoints + with the newly implemented `TypeSerializerSnapshot` for the serializer. + 4. Once you have a savepoint taken with Flink 1.7, the savepoint will contain `TypeSerializerSnapshot` as the + state serializer snapshot, and the serializer instance will no longer be written in the savepoint. + At this point, it is now safe to remove all implementations of the old abstraction (remove the old + `TypeSerializerConfigSnapshot` implementation as will as the + `TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)` from the serializer). {% top %} diff --git a/docs/dev/stream/state/index.md b/docs/dev/stream/state/index.md index fe725afc210f87..10de348a76d25c 100644 --- a/docs/dev/stream/state/index.md +++ b/docs/dev/stream/state/index.md @@ -49,8 +49,10 @@ Where to go next? ----------------- * [Working with State](state.html): Shows how to use state in a Flink application and explains the different kinds of state. +* [The Broadcast State Pattern](broadcast_state.html): Explains how to connect a broadcast stream with a non-broadcast stream and use state to exchange information between them. * [Checkpointing](checkpointing.html): Describes how to enable and configure checkpointing for fault tolerance. * [Queryable State](queryable_state.html): Explains how to access state from outside of Flink during runtime. -* [Custom Serialization for Managed State](custom_serialization.html): Discusses custom serialization logic for state and its upgrades. +* [State Schema Evolution](schema_evolution.html): Shows how schema of state types can be evolved. +* [Custom Serialization for Managed State](custom_serialization.html): Discusses how to implement custom serializers, especially for schema evolution. {% top %} diff --git a/docs/dev/stream/state/queryable_state.md b/docs/dev/stream/state/queryable_state.md index af646dfa7a49bf..7963487e6cd83d 100644 --- a/docs/dev/stream/state/queryable_state.md +++ b/docs/dev/stream/state/queryable_state.md @@ -1,7 +1,7 @@ --- title: "Queryable State" nav-parent_id: streaming_state -nav-pos: 3 +nav-pos: 4 is_beta: true --- + +* ToC +{:toc} + +Apache Flink streaming applications are typically designed to run indefinitely or for long periods of time. +As with all long-running services, the applications need to be updated to adapt to changing requirements. +This goes the same for data schemas that the applications work against; they evolve along with the application. + +This page provides an overview of how you can evolve your state type's data schema. +The current restrictions varies across different types and state structures (`ValueState`, `ListState`, etc.). + +Note that the information on this page is relevant only if you are using state serializers that are +generated by Flink's own [type serialization framework]({{ site.baseurl }}/dev/types_serialization.html). +That is, when declaring your state, the provided state descriptor is not configured to use a specific `TypeSerializer` +or `TypeInformation`, in which case Flink infers information about the state type: + +
    +{% highlight java %} +ListStateDescriptor descriptor = + new ListStateDescriptor<>( + "state-name", + MyPojoType.class); + +checkpointedState = getRuntimeContext().getListState(descriptor); +{% endhighlight %} +
    + +Under the hood, whether or not the schema of state can be evolved depends on the serializer used to read / write +persisted state bytes. Simply put, a registered state's schema can only be evolved if its serializer properly +supports it. This is handled transparently by serializers generated by Flink's type serialization framework +(current scope of support is listed [below]({{ site.baseurl }}/dev/stream/state/schema_evolution.html#supported-data-types-for-schema-evolution)). + +If you intend to implement a custom `TypeSerializer` for your state type and would like to learn how to implement +the serializer to support state schema evolution, please refer to +[Custom State Serialization]({{ site.baseurl }}/dev/stream/state/custom_serialization.html). +The documentation there also covers necessary internal details about the interplay between state serializers and Flink's +state backends to support state schema evolution. + +## Evolving state schema + +To evolve the schema of a given state type, you would take the following steps: + + 1. Take a savepoint of your Flink streaming job. + 2. Update state types in your application (e.g., modifying your Avro type schema). + 3. Restore the job from the savepoint. When accessing state for the first time, Flink will assess whether or not + the schema had been changed for the state, and migrate state schema if necessary. + +The process of migrating state to adapt to changed schemas happens automatically, and independently for each state. +This process is performed internally by Flink by first checking if the new serializer for the state has different +serialization schema than the previous serializer; if so, the previous serializer is used to read the state to objects, +and written back to bytes again with the new serializer. + +Further details about the migration process is out of the scope of this documentation; please refer to +[here]({{ site.baseurl }}/dev/stream/state/custom_serialization.html). + +## Supported data types for schema evolution + +Currently, schema evolution is supported only for Avro. Therefore, if you care about schema evolution for +state, it is currently recommended to always use Avro for state data types. + +There are plans to extend the support for more composite types, such as POJOs; for more details, +please refer to [FLINK-10897](https://issues.apache.org/jira/browse/FLINK-10897). + +### Avro types + +Flink fully supports evolving schema of Avro type state, as long as the schema change is considered compatible by +[Avro's rules for schema resolution](http://avro.apache.org/docs/current/spec.html#Schema+Resolution). + +One limitation is that Avro generated classes used as the state type cannot be relocated or have different +namespaces when the job is restored. + +{% top %} diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md index 3ea9b7a7b0e931..1654c9a77beb66 100644 --- a/docs/dev/stream/state/state.md +++ b/docs/dev/stream/state/state.md @@ -142,7 +142,7 @@ is available in a `RichFunction` has these methods for accessing state: * `ValueState getState(ValueStateDescriptor)` * `ReducingState getReducingState(ReducingStateDescriptor)` * `ListState getListState(ListStateDescriptor)` -* `AggregatingState getAggregatingState(AggregatingState)` +* `AggregatingState getAggregatingState(AggregatingStateDescriptor)` * `FoldingState getFoldingState(FoldingStateDescriptor)` * `MapState getMapState(MapStateDescriptor)` @@ -266,6 +266,136 @@ a `ValueState`. Once the count reaches 2 it will emit the average and clear the we start over from `0`. Note that this would keep a different state value for each different input key if we had tuples with different values in the first field. +### State Time-To-Live (TTL) + +A *time-to-live* (TTL) can be assigned to the keyed state of any type. If a TTL is configured and a +state value has expired, the stored value will be cleaned up on a best effort basis which is +discussed in more detail below. + +All state collection types support per-entry TTLs. This means that list elements and map entries +expire independently. + +In order to use state TTL one must first build a `StateTtlConfig` configuration object. The TTL +functionality can then be enabled in any state descriptor by passing the configuration: + +
    +
    +{% highlight java %} +import org.apache.flink.api.common.state.StateTtlConfig; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.time.Time; + +StateTtlConfig ttlConfig = StateTtlConfig + .newBuilder(Time.seconds(1)) + .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) + .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) + .build(); + +ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("text state", String.class); +stateDescriptor.enableTimeToLive(ttlConfig); +{% endhighlight %} +
    + +
    +{% highlight scala %} +import org.apache.flink.api.common.state.StateTtlConfig +import org.apache.flink.api.common.state.ValueStateDescriptor +import org.apache.flink.api.common.time.Time + +val ttlConfig = StateTtlConfig + .newBuilder(Time.seconds(1)) + .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) + .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) + .build + +val stateDescriptor = new ValueStateDescriptor[String]("text state", classOf[String]) +stateDescriptor.enableTimeToLive(ttlConfig) +{% endhighlight %} +
    +
    + +The configuration has several options to consider: + +The first parameter of the `newBuilder` method is mandatory, it is the time-to-live value. + +The update type configures when the state TTL is refreshed (by default `OnCreateAndWrite`): + + - `StateTtlConfig.UpdateType.OnCreateAndWrite` - only on creation and write access + - `StateTtlConfig.UpdateType.OnReadAndWrite` - also on read access + +The state visibility configures whether the expired value is returned on read access +if it is not cleaned up yet (by default `NeverReturnExpired`): + + - `StateTtlConfig.StateVisibility.NeverReturnExpired` - expired value is never returned + - `StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp` - returned if still available + +In case of `NeverReturnExpired`, the expired state behaves as if it does not exist anymore, +even if it still has to be removed. The option can be useful for use cases +where data has to become unavailable for read access strictly after TTL, +e.g. application working with privacy sensitive data. + +Another option `ReturnExpiredIfNotCleanedUp` allows to return the expired state before its cleanup. + +**Notes:** + +- The state backends store the timestamp of the last modification along with the user value, +which means that enabling this feature increases consumption of state storage. +Heap state backend stores an additional Java object with a reference to the user state object +and a primitive long value in memory. The RocksDB state backend adds 8 bytes per stored value, list entry or map entry. + +- Only TTLs in reference to *processing time* are currently supported. + +- Trying to restore state, which was previously configured without TTL, using TTL enabled descriptor or vice versa +will lead to compatibility failure and `StateMigrationException`. + +- The TTL configuration is not part of check- or savepoints but rather a way of how Flink treats it in the currently running job. + +- The map state with TTL currently supports null user values only if the user value serializer can handle null values. +If the serializer does not support null values, it can be wrapped with `NullableSerializer` at the cost of an extra byte in the serialized form. + +#### Cleanup of Expired State + +Currently, expired values are only removed when they are read out explicitly, +e.g. by calling `ValueState.value()`. + +Attention This means that by default if expired state is not read, +it won't be removed, possibly leading to ever growing state. This might change in future releases. + +Additionally, you can activate the cleanup at the moment of taking the full state snapshot which +will reduce its size. The local state is not cleaned up under the current implementation +but it will not include the removed expired state in case of restoration from the previous snapshot. +It can be configured in `StateTtlConfig`: + +
    +
    +{% highlight java %} +import org.apache.flink.api.common.state.StateTtlConfig; +import org.apache.flink.api.common.time.Time; + +StateTtlConfig ttlConfig = StateTtlConfig + .newBuilder(Time.seconds(1)) + .cleanupFullSnapshot() + .build(); +{% endhighlight %} +
    + +
    +{% highlight scala %} +import org.apache.flink.api.common.state.StateTtlConfig +import org.apache.flink.api.common.time.Time + +val ttlConfig = StateTtlConfig + .newBuilder(Time.seconds(1)) + .cleanupFullSnapshot + .build +{% endhighlight %} +
    +
    + +This option is not applicable for the incremental checkpointing in the RocksDB state backend. + +More strategies will be added in the future for cleaning up expired state automatically in the background. + ### State in the Scala DataStream API In addition to the interface described above, the Scala API has shortcuts for stateful @@ -345,7 +475,7 @@ public class BufferingSink } @Override - public void invoke(Tuple2 value) throws Exception { + public void invoke(Tuple2 value, Context contex) throws Exception { bufferedElements.add(value); if (bufferedElements.size() == threshold) { for (Tuple2 element: bufferedElements) { @@ -368,7 +498,7 @@ public class BufferingSink ListStateDescriptor> descriptor = new ListStateDescriptor<>( "buffered-elements", - TypeInformation.of(new TypeHint>() {})); + TypeInformation.of(new TypeHint>() {})); checkpointedState = context.getOperatorStateStore().getListState(descriptor); @@ -386,15 +516,14 @@ public class BufferingSink {% highlight scala %} class BufferingSink(threshold: Int = 0) extends SinkFunction[(String, Int)] - with CheckpointedFunction - with CheckpointedRestoring[List[(String, Int)]] { + with CheckpointedFunction { @transient private var checkpointedState: ListState[(String, Int)] = _ private val bufferedElements = ListBuffer[(String, Int)]() - override def invoke(value: (String, Int)): Unit = { + override def invoke(value: (String, Int), context: Context): Unit = { bufferedElements += value if (bufferedElements.size == threshold) { for (element <- bufferedElements) { @@ -426,9 +555,6 @@ class BufferingSink(threshold: Int = 0) } } - override def restoreState(state: List[(String, Int)]): Unit = { - bufferedElements ++= state - } } {% endhighlight %}
    @@ -514,7 +640,7 @@ public static class CounterSource implements ListCheckpointed { /** current offset for exactly once semantics */ - private Long offset; + private Long offset = 0L; /** flag for job cancellation */ private volatile boolean isRunning = true; diff --git a/docs/dev/stream/testing.md b/docs/dev/stream/testing.md index 0a43d313e6dbd3..7676b199c22259 100644 --- a/docs/dev/stream/testing.md +++ b/docs/dev/stream/testing.md @@ -140,7 +140,7 @@ You could write the following integration test:
    {% highlight java %} -public class ExampleIntegrationTest extends StreamingMultipleProgramsTestBase { +public class ExampleIntegrationTest extends AbstractTestBase { @Test public void testMultiply() throws Exception { @@ -181,7 +181,7 @@ public class ExampleIntegrationTest extends StreamingMultipleProgramsTestBase {
    {% highlight scala %} -class ExampleIntegrationTest extends StreamingMultipleProgramsTestBase { +class ExampleIntegrationTest extends AbstractTestBase { @Test def testMultiply(): Unit = { diff --git a/docs/dev/table/common.md b/docs/dev/table/common.md index e1d380942c8ea3..5d2521f4546baa 100644 --- a/docs/dev/table/common.md +++ b/docs/dev/table/common.md @@ -46,6 +46,8 @@ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); tableEnv.registerTable("table1", ...) // or tableEnv.registerTableSource("table2", ...); // or tableEnv.registerExternalCatalog("extCat", ...); +// register an output Table +tableEnv.registerTableSink("outputTable", ...); // create a Table from a Table API query Table tapiResult = tableEnv.scan("table1").select(...); @@ -53,7 +55,7 @@ Table tapiResult = tableEnv.scan("table1").select(...); Table sqlResult = tableEnv.sqlQuery("SELECT ... FROM table2 ... "); // emit a Table API result Table to a TableSink, same for SQL result -tapiResult.writeToSink(...); +tapiResult.insertInto("outputTable"); // execute env.execute(); @@ -72,7 +74,9 @@ val tableEnv = TableEnvironment.getTableEnvironment(env) // register a Table tableEnv.registerTable("table1", ...) // or tableEnv.registerTableSource("table2", ...) // or -tableEnv.registerExternalCatalog("extCat", ...) +tableEnv.registerExternalCatalog("extCat", ...) +// register an output Table +tableEnv.registerTableSink("outputTable", ...); // create a Table from a Table API query val tapiResult = tableEnv.scan("table1").select(...) @@ -80,7 +84,7 @@ val tapiResult = tableEnv.scan("table1").select(...) val sqlResult = tableEnv.sqlQuery("SELECT ... FROM table2 ...") // emit a Table API result Table to a TableSink, same for SQL result -tapiResult.writeToSink(...) +tapiResult.insertInto("outputTable") // execute env.execute() @@ -173,7 +177,7 @@ A `Table` is registered in a `TableEnvironment` as follows: StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); // Table is the result of a simple projection query -Table projTable = tableEnv.scan("X").project(...); +Table projTable = tableEnv.scan("X").select(...); // register the Table projTable as table "projectedX" tableEnv.registerTable("projectedTable", projTable); @@ -186,7 +190,7 @@ tableEnv.registerTable("projectedTable", projTable); val tableEnv = TableEnvironment.getTableEnvironment(env) // Table is the result of a simple projection query -val projTable: Table = tableEnv.scan("X").project(...) +val projTable: Table = tableEnv.scan("X").select(...) // register the Table projTable as table "projectedX" tableEnv.registerTable("projectedTable", projTable) @@ -365,9 +369,9 @@ val tableEnv = TableEnvironment.getTableEnvironment(env) // register Orders table // scan registered Orders table -Table orders = tableEnv.scan("Orders") +val orders = tableEnv.scan("Orders") // compute revenue for all customers from France -Table revenue = orders +val revenue = orders .filter('cCountry === "FRANCE") .groupBy('cID, 'cName) .select('cID, 'cName, 'revenue.sum AS 'revSum) @@ -419,7 +423,7 @@ val tableEnv = TableEnvironment.getTableEnvironment(env) // register Orders table // compute revenue for all customers from France -Table revenue = tableEnv.sqlQuery(""" +val revenue = tableEnv.sqlQuery(""" |SELECT cID, cName, SUM(revenue) AS revSum |FROM Orders |WHERE cCountry = 'FRANCE' @@ -500,10 +504,7 @@ A batch `Table` can only be written to a `BatchTableSink`, while a streaming `Ta Please see the documentation about [Table Sources & Sinks]({{ site.baseurl }}/dev/table/sourceSinks.html) for details about available sinks and instructions for how to implement a custom `TableSink`. -There are two ways to emit a table: - -1. The `Table.writeToSink(TableSink sink)` method emits the table using the provided `TableSink` and automatically configures the sink with the schema of the table to emit. -2. The `Table.insertInto(String sinkTable)` method looks up a `TableSink` that was registered with a specific schema under the provided name in the `TableEnvironment`'s catalog. The schema of the table to emit is validated against the schema of the registered `TableSink`. +The `Table.insertInto(String tableName)` method emits the `Table` to a registered `TableSink`. The method looks up the `TableSink` from the catalog by the name and validates that the schema of the `Table` is identical to the schema of the `TableSink`. The following examples shows how to emit a `Table`: @@ -513,22 +514,17 @@ The following examples shows how to emit a `Table`: // get a StreamTableEnvironment, works for BatchTableEnvironment equivalently StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); -// compute a result Table using Table API operators and/or SQL queries -Table result = ... - // create a TableSink TableSink sink = new CsvTableSink("/path/to/file", fieldDelim = "|"); -// METHOD 1: -// Emit the result Table to the TableSink via the writeToSink() method -result.writeToSink(sink); - -// METHOD 2: -// Register the TableSink with a specific schema +// register the TableSink with a specific schema String[] fieldNames = {"a", "b", "c"}; TypeInformation[] fieldTypes = {Types.INT, Types.STRING, Types.LONG}; tableEnv.registerTableSink("CsvSinkTable", fieldNames, fieldTypes, sink); -// Emit the result Table to the registered TableSink via the insertInto() method + +// compute a result Table using Table API operators and/or SQL queries +Table result = ... +// emit the result Table to the registered TableSink result.insertInto("CsvSinkTable"); // execute the program @@ -540,22 +536,18 @@ result.insertInto("CsvSinkTable"); // get a TableEnvironment val tableEnv = TableEnvironment.getTableEnvironment(env) -// compute a result Table using Table API operators and/or SQL queries -val result: Table = ... - // create a TableSink val sink: TableSink = new CsvTableSink("/path/to/file", fieldDelim = "|") -// METHOD 1: -// Emit the result Table to the TableSink via the writeToSink() method -result.writeToSink(sink) - -// METHOD 2: -// Register the TableSink with a specific schema +// register the TableSink with a specific schema val fieldNames: Array[String] = Array("a", "b", "c") val fieldTypes: Array[TypeInformation] = Array(Types.INT, Types.STRING, Types.LONG) tableEnv.registerTableSink("CsvSinkTable", fieldNames, fieldTypes, sink) -// Emit the result Table to the registered TableSink via the insertInto() method + +// compute a result Table using Table API operators and/or SQL queries +val result: Table = ... + +// emit the result Table to the registered TableSink result.insertInto("CsvSinkTable") // execute the program @@ -576,7 +568,7 @@ Table API and SQL queries are translated into [DataStream]({{ site.baseurl }}/de A Table API or SQL query is translated when: -* a `Table` is emitted to a `TableSink`, i.e., when `Table.writeToSink()` or `Table.insertInto()` is called. +* a `Table` is emitted to a `TableSink`, i.e., when `Table.insertInto()` is called. * a SQL update query is specified, i.e., when `TableEnvironment.sqlUpdate()` is called. * a `Table` is converted into a `DataStream` or `DataSet` (see [Integration with DataStream and DataSet API](#integration-with-dataStream-and-dataSet-api)). @@ -753,7 +745,7 @@ val retractStream: DataStream[(Boolean, Row)] = tableEnv.toRetractStream[Row](ta
    -**Note:** A detailed discussion about dynamic tables and their properties is given in the [Streaming Queries]({{ site.baseurl }}/dev/table/streaming.html) document. +**Note:** A detailed discussion about dynamic tables and their properties is given in the [Dynamic Tables](streaming/dynamic_tables.html) document. #### Convert a Table into a DataSet @@ -775,8 +767,8 @@ DataSet dsRow = tableEnv.toDataSet(table, Row.class); TupleTypeInfo> tupleType = new TupleTypeInfo<>( Types.STRING(), Types.INT()); -DataStream> dsTuple = - tableEnv.toAppendStream(table, tupleType); +DataSet> dsTuple = + tableEnv.toDataSet(table, tupleType); {% endhighlight %}
    @@ -808,7 +800,7 @@ The mapping of a data type to a table schema can happen in two ways: **based on **Position-based Mapping** -Position-based mapping can be used to give fields a more meaningful name while keeping the field order. This mapping is available for composite data types *with a defined field order* as well as atomic types. Composite data types such as tuples, rows, and case classes have such a field order. However, fields of a POJO must be mapped based on the field names (see next section). +Position-based mapping can be used to give fields a more meaningful name while keeping the field order. This mapping is available for composite data types *with a defined field order* as well as atomic types. Composite data types such as tuples, rows, and case classes have such a field order. However, fields of a POJO must be mapped based on the field names (see next section). Fields can be projected out but can't be renamed using an alias `as`. When defining a position-based mapping, the specified names must not exist in the input data type, otherwise the API will assume that the mapping should happen based on the field names. If no field names are specified, the default field names and field order of the composite type are used or `f0` for atomic types. @@ -823,6 +815,9 @@ DataStream> stream = ... // convert DataStream into Table with default field names "f0" and "f1" Table table = tableEnv.fromDataStream(stream); +// convert DataStream into Table with field "myLong" only +Table table = tableEnv.fromDataStream(stream, "myLong"); + // convert DataStream into Table with field names "myLong" and "myInt" Table table = tableEnv.fromDataStream(stream, "myLong, myInt"); {% endhighlight %} @@ -838,8 +833,11 @@ val stream: DataStream[(Long, Int)] = ... // convert DataStream into Table with default field names "_1" and "_2" val table: Table = tableEnv.fromDataStream(stream) +// convert DataStream into Table with field "myLong" only +val table: Table = tableEnv.fromDataStream(stream, 'myLong) + // convert DataStream into Table with field names "myLong" and "myInt" -val table: Table = tableEnv.fromDataStream(stream, 'myLong 'myInt) +val table: Table = tableEnv.fromDataStream(stream, 'myLong, 'myInt) {% endhighlight %}
    diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md new file mode 100644 index 00000000000000..27c6920fcd73f5 --- /dev/null +++ b/docs/dev/table/connect.md @@ -0,0 +1,1205 @@ +--- +title: "Connect to External Systems" +nav-parent_id: tableapi +nav-pos: 19 +--- + + +Flink's Table API & SQL programs can be connected to other external systems for reading and writing both batch and streaming tables. A table source provides access to data which is stored in external systems (such as a database, key-value store, message queue, or file system). A table sink emits a table to an external storage system. Depending on the type of source and sink, they support different formats such as CSV, Parquet, or ORC. + +This page describes how to declare built-in table sources and/or table sinks and register them in Flink. After a source or sink has been registered, it can be accessed by Table API & SQL statements. + +Attention If you want to implement your own *custom* table source or sink, have a look at the [user-defined sources & sinks page](sourceSinks.html). + +* This will be replaced by the TOC +{:toc} + +Dependencies +------------ + +The following tables list all available connectors and formats. Their mutual compatibility is tagged in the corresponding sections for [table connectors](connect.html#table-connectors) and [table formats](connect.html#table-formats). The following tables provide dependency information for both projects using a build automation tool (such as Maven or SBT) and SQL Client with SQL JAR bundles. + +{% if site.is_stable %} + +### Connectors + +| Name | Version | Maven dependency | SQL Client JAR | +| :---------------- | :------------------ | :--------------------------- | :----------------------| +| Filesystem | | Built-in | Built-in | +| Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.8 | `flink-connector-kafka-0.8` | Not available | +| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.11 | `flink-connector-kafka-0.11` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.11{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.11{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.11+ (`universal`) | `flink-connector-kafka` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka{{site.scala_version_suffix}}-{{site.version}}.jar) | + +### Formats + +| Name | Maven dependency | SQL Client JAR | +| :---------------- | :--------------------------- | :--------------------- | +| CSV | Built-in | Built-in | +| JSON | `flink-json` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) | +| Apache Avro | `flink-avro` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) | + +{% else %} + +These tables are only available for stable releases. + +{% endif %} + +{% top %} + +Overview +-------- + +Beginning from Flink 1.6, the declaration of a connection to an external system is separated from the actual implementation. + +Connections can be specified either + +- **programmatically** using a `Descriptor` under `org.apache.flink.table.descriptors` for Table & SQL API +- or **declaratively** via [YAML configuration files](http://yaml.org/) for the SQL Client. + +This allows not only for better unification of APIs and SQL Client but also for better extensibility in case of [custom implementations](sourceSinks.html) without changing the actual declaration. + +Every declaration is similar to a SQL `CREATE TABLE` statement. One can define the name of the table, the schema of the table, a connector, and a data format upfront for connecting to an external system. + +The **connector** describes the external system that stores the data of a table. Storage systems such as [Apacha Kafka](http://kafka.apache.org/) or a regular file system can be declared here. The connector might already provide a fixed format with fields and schema. + +Some systems support different **data formats**. For example, a table that is stored in Kafka or in files can encode its rows with CSV, JSON, or Avro. A database connector might need the table schema here. Whether or not a storage system requires the definition of a format, is documented for every [connector](connect.html#table-connectors). Different systems also require different [types of formats](connect.html#table-formats) (e.g., column-oriented formats vs. row-oriented formats). The documentation states which format types and connectors are compatible. + +The **table schema** defines the schema of a table that is exposed to SQL queries. It describes how a source maps the data format to the table schema and a sink vice versa. The schema has access to fields defined by the connector or format. It can use one or more fields for extracting or inserting [time attributes](streaming/time_attributes.html). If input fields have no deterministic field order, the schema clearly defines column names, their order, and origin. + +The subsequent sections will cover each definition part ([connector](connect.html#table-connectors), [format](connect.html#table-formats), and [schema](connect.html#table-schema)) in more detail. The following example shows how to pass them: + +
    +
    +{% highlight java %} +tableEnvironment + .connect(...) + .withFormat(...) + .withSchema(...) + .inAppendMode() + .registerTableSource("MyTable") +{% endhighlight %} +
    + +
    +{% highlight yaml %} +name: MyTable +type: source +update-mode: append +connector: ... +format: ... +schema: ... +{% endhighlight %} +
    +
    + +The table's type (`source`, `sink`, or `both`) determines how a table is registered. In case of table type `both`, both a table source and table sink are registered under the same name. Logically, this means that we can both read and write to such a table similarly to a table in a regular DBMS. + +For streaming queries, an [update mode](connect.html#update-mode) declares how to communicate between a dynamic table and the storage system for continuous queries. + +The following code shows a full example of how to connect to Kafka for reading Avro records. + +
    +
    +{% highlight java %} +tableEnvironment + // declare the external system to connect to + .connect( + new Kafka() + .version("0.10") + .topic("test-input") + .startFromEarliest() + .property("zookeeper.connect", "localhost:2181") + .property("bootstrap.servers", "localhost:9092") + ) + + // declare a format for this system + .withFormat( + new Avro() + .avroSchema( + "{" + + " \"namespace\": \"org.myorganization\"," + + " \"type\": \"record\"," + + " \"name\": \"UserMessage\"," + + " \"fields\": [" + + " {\"name\": \"timestamp\", \"type\": \"string\"}," + + " {\"name\": \"user\", \"type\": \"long\"}," + + " {\"name\": \"message\", \"type\": [\"string\", \"null\"]}" + + " ]" + + "}" + ) + ) + + // declare the schema of the table + .withSchema( + new Schema() + .field("rowtime", Types.SQL_TIMESTAMP) + .rowtime(new Rowtime() + .timestampsFromField("timestamp") + .watermarksPeriodicBounded(60000) + ) + .field("user", Types.LONG) + .field("message", Types.STRING) + ) + + // specify the update-mode for streaming tables + .inAppendMode() + + // register as source, sink, or both and under a name + .registerTableSource("MyUserTable"); +{% endhighlight %} +
    + +
    +{% highlight yaml %} +tables: + - name: MyUserTable # name the new table + type: source # declare if the table should be "source", "sink", or "both" + update-mode: append # specify the update-mode for streaming tables + + # declare the external system to connect to + connector: + type: kafka + version: "0.10" + topic: test-input + startup-mode: earliest-offset + properties: + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + + # declare a format for this system + format: + type: avro + avro-schema: > + { + "namespace": "org.myorganization", + "type": "record", + "name": "UserMessage", + "fields": [ + {"name": "ts", "type": "string"}, + {"name": "user", "type": "long"}, + {"name": "message", "type": ["string", "null"]} + ] + } + + # declare the schema of the table + schema: + - name: rowtime + type: TIMESTAMP + rowtime: + timestamps: + type: from-field + from: ts + watermarks: + type: periodic-bounded + delay: "60000" + - name: user + type: BIGINT + - name: message + type: VARCHAR +{% endhighlight %} +
    +
    + +In both ways the desired connection properties are converted into normalized, string-based key-value pairs. So-called [table factories](sourceSinks.html#define-a-tablefactory) create configured table sources, table sinks, and corresponding formats from the key-value pairs. All table factories that can be found via Java's [Service Provider Interfaces (SPI)](https://docs.oracle.com/javase/tutorial/sound/SPI-intro.html) are taken into account when searching for exactly-one matching table factory. + +If no factory can be found or multiple factories match for the given properties, an exception will be thrown with additional information about considered factories and supported properties. + +{% top %} + +Table Schema +------------ + +The table schema defines the names and types of columns similar to the column definitions of a SQL `CREATE TABLE` statement. In addition, one can specify how columns are mapped from and to fields of the format in which the table data is encoded. The origin of a field might be important if the name of the column should differ from the input/output format. For instance, a column `user_name` should reference the field `$$-user-name` from a JSON format. Additionally, the schema is needed to map types from an external system to Flink's representation. In case of a table sink, it ensures that only data with valid schema is written to an external system. + +The following example shows a simple schema without time attributes and one-to-one field mapping of input/output to table columns. + +
    +
    +{% highlight java %} +.withSchema( + new Schema() + .field("MyField1", Types.INT) // required: specify the fields of the table (in this order) + .field("MyField2", Types.STRING) + .field("MyField3", Types.BOOLEAN) +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +schema: + - name: MyField1 # required: specify the fields of the table (in this order) + type: INT + - name: MyField2 + type: VARCHAR + - name: MyField3 + type: BOOLEAN +{% endhighlight %} +
    +
    + +For *each field*, the following properties can be declared in addition to the column's name and type: + +
    +
    +{% highlight java %} +.withSchema( + new Schema() + .field("MyField1", Types.SQL_TIMESTAMP) + .proctime() // optional: declares this field as a processing-time attribute + .field("MyField2", Types.SQL_TIMESTAMP) + .rowtime(...) // optional: declares this field as a event-time attribute + .field("MyField3", Types.BOOLEAN) + .from("mf3") // optional: original field in the input that is referenced/aliased by this field +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +schema: + - name: MyField1 + type: TIMESTAMP + proctime: true # optional: boolean flag whether this field should be a processing-time attribute + - name: MyField2 + type: TIMESTAMP + rowtime: ... # optional: wether this field should be a event-time attribute + - name: MyField3 + type: BOOLEAN + from: mf3 # optional: original field in the input that is referenced/aliased by this field +{% endhighlight %} +
    +
    + +Time attributes are essential when working with unbounded streaming tables. Therefore both processing-time and event-time (also known as "rowtime") attributes can be defined as part of the schema. + +For more information about time handling in Flink and especially event-time, we recommend the general [event-time section](streaming/time_attributes.html). + +### Rowtime Attributes + +In order to control the event-time behavior for tables, Flink provides predefined timestamp extractors and watermark strategies. + +The following timestamp extractors are supported: + +
    +
    +{% highlight java %} +// Converts an existing LONG or SQL_TIMESTAMP field in the input into the rowtime attribute. +.rowtime( + new Rowtime() + .timestampsFromField("ts_field") // required: original field name in the input +) + +// Converts the assigned timestamps from a DataStream API record into the rowtime attribute +// and thus preserves the assigned timestamps from the source. +// This requires a source that assigns timestamps (e.g., Kafka 0.10+). +.rowtime( + new Rowtime() + .timestampsFromSource() +) + +// Sets a custom timestamp extractor to be used for the rowtime attribute. +// The extractor must extend `org.apache.flink.table.sources.tsextractors.TimestampExtractor`. +.rowtime( + new Rowtime() + .timestampsFromExtractor(...) +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +# Converts an existing BIGINT or TIMESTAMP field in the input into the rowtime attribute. +rowtime: + timestamps: + type: from-field + from: "ts_field" # required: original field name in the input + +# Converts the assigned timestamps from a DataStream API record into the rowtime attribute +# and thus preserves the assigned timestamps from the source. +rowtime: + timestamps: + type: from-source +{% endhighlight %} +
    +
    + +The following watermark strategies are supported: + +
    +
    +{% highlight java %} +// Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum +// observed timestamp so far minus 1. Rows that have a timestamp equal to the max timestamp +// are not late. +.rowtime( + new Rowtime() + .watermarksPeriodicAscending() +) + +// Sets a built-in watermark strategy for rowtime attributes which are out-of-order by a bounded time interval. +// Emits watermarks which are the maximum observed timestamp minus the specified delay. +.rowtime( + new Rowtime() + .watermarksPeriodicBounded(2000) // delay in milliseconds +) + +// Sets a built-in watermark strategy which indicates the watermarks should be preserved from the +// underlying DataStream API and thus preserves the assigned watermarks from the source. +.rowtime( + new Rowtime() + .watermarksFromSource() +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +# Sets a watermark strategy for ascending rowtime attributes. Emits a watermark of the maximum +# observed timestamp so far minus 1. Rows that have a timestamp equal to the max timestamp +# are not late. +rowtime: + watermarks: + type: periodic-ascending + +# Sets a built-in watermark strategy for rowtime attributes which are out-of-order by a bounded time interval. +# Emits watermarks which are the maximum observed timestamp minus the specified delay. +rowtime: + watermarks: + type: periodic-bounded + delay: ... # required: delay in milliseconds + +# Sets a built-in watermark strategy which indicates the watermarks should be preserved from the +# underlying DataStream API and thus preserves the assigned watermarks from the source. +rowtime: + watermarks: + type: from-source +{% endhighlight %} +
    +
    + +Make sure to always declare both timestamps and watermarks. Watermarks are required for triggering time-based operations. + +### Type Strings + +Because type information is only available in a programming language, the following type strings are supported for being defined in a YAML file: + +{% highlight yaml %} +VARCHAR +BOOLEAN +TINYINT +SMALLINT +INT +BIGINT +FLOAT +DOUBLE +DECIMAL +DATE +TIME +TIMESTAMP +MAP # generic map; e.g. MAP that is mapped to Flink's MapTypeInfo +MULTISET # multiset; e.g. MULTISET that is mapped to Flink's MultisetTypeInfo +PRIMITIVE_ARRAY # primitive array; e.g. PRIMITIVE_ARRAY that is mapped to Flink's PrimitiveArrayTypeInfo +OBJECT_ARRAY # object array; e.g. OBJECT_ARRAY that is mapped to + # Flink's ObjectArrayTypeInfo +ROW # unnamed row; e.g. ROW that is mapped to Flink's RowTypeInfo + # with indexed fields names f0, f1, ... +ROW # named row; e.g., ROW that + # is mapped to Flink's RowTypeInfo +POJO # e.g., POJO that is mapped to Flink's PojoTypeInfo +ANY # e.g., ANY that is mapped to Flink's GenericTypeInfo +ANY # used for type information that is not supported by Flink's Table & SQL API +{% endhighlight %} + +{% top %} + +Update Modes +------------ + +For streaming queries, it is required to declare how to perform the [conversion between a dynamic table and an external connector](streaming/dynamic_tables.html#continuous-queries). The *update mode* specifies which kind of messages should be exchanged with the external system: + +**Append Mode:** In append mode, a dynamic table and an external connector only exchange INSERT messages. + +**Retract Mode:** In retract mode, a dynamic table and an external connector exchange ADD and RETRACT messages. An INSERT change is encoded as an ADD message, a DELETE change as a RETRACT message, and an UPDATE change as a RETRACT message for the updated (previous) row and an ADD message for the updating (new) row. In this mode, a key must not be defined as opposed to upsert mode. However, every update consists of two messages which is less efficient. + +**Upsert Mode:** In upsert mode, a dynamic table and an external connector exchange UPSERT and DELETE messages. This mode requires a (possibly composite) unique key by which updates can be propagated. The external connector needs to be aware of the unique key attribute in order to apply messages correctly. INSERT and UPDATE changes are encoded as UPSERT messages. DELETE changes as DELETE messages. The main difference to a retract stream is that UPDATE changes are encoded with a single message and are therefore more efficient. + +Attention The documentation of each connector states which update modes are supported. + +
    +
    +{% highlight java %} +.connect(...) + .inAppendMode() // otherwise: inUpsertMode() or inRetractMode() +{% endhighlight %} +
    + +
    +{% highlight yaml %} +tables: + - name: ... + update-mode: append # otherwise: "retract" or "upsert" +{% endhighlight %} +
    +
    + +See also the [general streaming concepts documentation](streaming/dynamic_tables.html#continuous-queries) for more information. + +{% top %} + +Table Connectors +---------------- + +Flink provides a set of connectors for connecting to external systems. + +Please note that not all connectors are available in both batch and streaming yet. Furthermore, not every streaming connector supports every streaming mode. Therefore, each connector is tagged accordingly. A format tag indicates that the connector requires a certain type of format. + +### File System Connector + +Source: Batch +Source: Streaming Append Mode +Sink: Batch +Sink: Streaming Append Mode +Format: CSV-only + +The file system connector allows for reading and writing from a local or distributed filesystem. A filesystem can be defined as: + +
    +
    +{% highlight java %} +.connect( + new FileSystem() + .path("file:///path/to/whatever") // required: path to a file or directory +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +connector: + type: filesystem + path: "file:///path/to/whatever" # required: path to a file or directory +{% endhighlight %} +
    +
    + +The file system connector itself is included in Flink and does not require an additional dependency. A corresponding format needs to be specified for reading and writing rows from and to a file system. + +Attention Make sure to include [Flink File System specific dependencies]({{ site.baseurl }}/ops/filesystems.html). + +Attention File system sources and sinks for streaming are only experimental. In the future, we will support actual streaming use cases, i.e., directory monitoring and bucket output. + +### Kafka Connector + +Source: Streaming Append Mode +Sink: Streaming Append Mode +Format: Serialization Schema +Format: Deserialization Schema + +The Kafka connector allows for reading and writing from and to an Apache Kafka topic. It can be defined as follows: + +
    +
    +{% highlight java %} +.connect( + new Kafka() + .version("0.11") // required: valid connector versions are + // "0.8", "0.9", "0.10", "0.11", and "universal" + .topic("...") // required: topic name from which the table is read + + // optional: connector specific properties + .property("zookeeper.connect", "localhost:2181") + .property("bootstrap.servers", "localhost:9092") + .property("group.id", "testGroup") + + // optional: select a startup mode for Kafka offsets + .startFromEarliest() + .startFromLatest() + .startFromSpecificOffsets(...) + + // optional: output partitioning from Flink's partitions into Kafka's partitions + .sinkPartitionerFixed() // each Flink partition ends up in at-most one Kafka partition (default) + .sinkPartitionerRoundRobin() // a Flink partition is distributed to Kafka partitions round-robin + .sinkPartitionerCustom(MyCustom.class) // use a custom FlinkKafkaPartitioner subclass +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +connector: + type: kafka + version: "0.11" # required: valid connector versions are + # "0.8", "0.9", "0.10", "0.11", and "universal" + topic: ... # required: topic name from which the table is read + + properties: # optional: connector specific properties + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + - key: group.id + value: testGroup + + startup-mode: ... # optional: valid modes are "earliest-offset", "latest-offset", + # "group-offsets", or "specific-offsets" + specific-offsets: # optional: used in case of startup mode with specific offsets + - partition: 0 + offset: 42 + - partition: 1 + offset: 300 + + sink-partitioner: ... # optional: output partitioning from Flink's partitions into Kafka's partitions + # valid are "fixed" (each Flink partition ends up in at most one Kafka partition), + # "round-robin" (a Flink partition is distributed to Kafka partitions round-robin) + # "custom" (use a custom FlinkKafkaPartitioner subclass) + sink-partitioner-class: org.mycompany.MyPartitioner # optional: used in case of sink partitioner custom +{% endhighlight %} +
    +
    + +**Specify the start reading position:** By default, the Kafka source will start reading data from the committed group offsets in Zookeeper or Kafka brokers. You can specify other start positions, which correspond to the configurations in section [Kafka Consumers Start Position Configuration]({{ site.baseurl }}/dev/connectors/kafka.html#kafka-consumers-start-position-configuration). + +**Flink-Kafka Sink Partitioning:** By default, a Kafka sink writes to at most as many partitions as its own parallelism (each parallel instance of the sink writes to exactly one partition). In order to distribute the writes to more partitions or control the routing of rows into partitions, a custom sink partitioner can be provided. The round-robin partitioner is useful to avoid an unbalanced partitioning. However, it will cause a lot of network connections between all the Flink instances and all the Kafka brokers. + +**Consistency guarantees:** By default, a Kafka sink ingests data with at-least-once guarantees into a Kafka topic if the query is executed with [checkpointing enabled]({{ site.baseurl }}/dev/stream/state/checkpointing.html#enabling-and-configuring-checkpointing). + +**Kafka 0.10+ Timestamps:** Since Kafka 0.10, Kafka messages have a timestamp as metadata that specifies when the record was written into the Kafka topic. These timestamps can be used for a [rowtime attribute](connect.html#defining-the-schema) by selecting `timestamps: from-source` in YAML and `timestampsFromSource()` in Java/Scala respectively. + +**Kafka 0.11+ Versioning:** Since Flink 1.7, the Kafka connector definition should be independent of a hard-coded Kafka version. Use the connector version `universal` as a wildcard for Flink's Kafka connector that is compatible with all Kafka versions starting from 0.11. + +Make sure to add the version-specific Kafka dependency. In addition, a corresponding format needs to be specified for reading and writing rows from and to Kafka. + +{% top %} + +### Elasticsearch Connector + +Sink: Streaming Append Mode +Sink: Streaming Upsert Mode +Format: JSON-only + +The Elasticsearch connector allows for writing into an index of the Elasticsearch search engine. + +The connector can operate in [upsert mode](#update-modes) for exchanging UPSERT/DELETE messages with the external system using a [key defined by the query](./streaming/dynamic_tables.html#table-to-stream-conversion). + +For append-only queries, the connector can also operate in [append mode](#update-modes) for exchanging only INSERT messages with the external system. If no key is defined by the query, a key is automatically generated by Elasticsearch. + +The connector can be defined as follows: + +
    +
    +{% highlight java %} +.connect( + new Elasticsearch() + .version("6") // required: valid connector versions are "6" + .host("localhost", 9200, "http") // required: one or more Elasticsearch hosts to connect to + .index("MyUsers") // required: Elasticsearch index + .documentType("user") // required: Elasticsearch document type + + .keyDelimiter("$") // optional: delimiter for composite keys ("_" by default) + // e.g., "$" would result in IDs "KEY1$KEY2$KEY3" + .keyNullLiteral("n/a") // optional: representation for null fields in keys ("null" by default) + + // optional: failure handling strategy in case a request to Elasticsearch fails (fail by default) + .failureHandlerFail() // optional: throws an exception if a request fails and causes a job failure + .failureHandlerIgnore() // or ignores failures and drops the request + .failureHandlerRetryRejected() // or re-adds requests that have failed due to queue capacity saturation + .failureHandlerCustom(...) // or custom failure handling with a ActionRequestFailureHandler subclass + + // optional: configure how to buffer elements before sending them in bulk to the cluster for efficiency + .disableFlushOnCheckpoint() // optional: disables flushing on checkpoint (see notes below!) + .bulkFlushMaxActions(42) // optional: maximum number of actions to buffer for each bulk request + .bulkFlushMaxSize("42 mb") // optional: maximum size of buffered actions in bytes per bulk request + // (only MB granularity is supported) + .bulkFlushInterval(60000L) // optional: bulk flush interval (in milliseconds) + + .bulkFlushBackoffConstant() // optional: use a constant backoff type + .bulkFlushBackoffExponential() // or use an exponential backoff type + .bulkFlushBackoffMaxRetries(3) // optional: maximum number of retries + .bulkFlushBackoffDelay(30000L) // optional: delay between each backoff attempt (in milliseconds) + + // optional: connection properties to be used during REST communication to Elasticsearch + .connectionMaxRetryTimeout(3) // optional: maximum timeout (in milliseconds) between retries + .connectionPathPrefix("/v1") // optional: prefix string to be added to every REST communication +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +connector: + type: elasticsearch + version: 6 # required: valid connector versions are "6" + hosts: # required: one or more Elasticsearch hosts to connect to + - hostname: "localhost" + port: 9200 + protocol: "http" + index: "MyUsers" # required: Elasticsearch index + document-type: "user" # required: Elasticsearch document type + + key-delimiter: "$" # optional: delimiter for composite keys ("_" by default) + # e.g., "$" would result in IDs "KEY1$KEY2$KEY3" + key-null-literal: "n/a" # optional: representation for null fields in keys ("null" by default) + + # optional: failure handling strategy in case a request to Elasticsearch fails ("fail" by default) + failure-handler: ... # valid strategies are "fail" (throws an exception if a request fails and + # thus causes a job failure), "ignore" (ignores failures and drops the request), + # "retry-rejected" (re-adds requests that have failed due to queue capacity + # saturation), or "custom" for failure handling with a + # ActionRequestFailureHandler subclass + + # optional: configure how to buffer elements before sending them in bulk to the cluster for efficiency + flush-on-checkpoint: true # optional: disables flushing on checkpoint (see notes below!) ("true" by default) + bulk-flush: + max-actions: 42 # optional: maximum number of actions to buffer for each bulk request + max-size: 42 mb # optional: maximum size of buffered actions in bytes per bulk request + # (only MB granularity is supported) + interval: 60000 # optional: bulk flush interval (in milliseconds) + back-off: # optional: backoff strategy ("disabled" by default) + type: ... # valid strategies are "disabled", "constant", or "exponential" + max-retries: 3 # optional: maximum number of retries + delay: 30000 # optional: delay between each backoff attempt (in milliseconds) + + # optional: connection properties to be used during REST communication to Elasticsearch + connection-max-retry-timeout: 3 # optional: maximum timeout (in milliseconds) between retries + connection-path-prefix: "/v1" # optional: prefix string to be added to every REST communication +{% endhighlight %} +
    +
    + +**Bulk flushing:** For more information about characteristics of the optional flushing parameters see the [corresponding low-level documentation]({{ site.baseurl }}/dev/connectors/elasticsearch.html). + +**Disabling flushing on checkpoint:** When disabled, a sink will not wait for all pending action requests to be acknowledged by Elasticsearch on checkpoints. Thus, a sink does NOT provide any strong guarantees for at-least-once delivery of action requests. + +**Key extraction:** Flink automatically extracts valid keys from a query. For example, a query `SELECT a, b, c FROM t GROUP BY a, b` defines a composite key of the fields `a` and `b`. The Elasticsearch connector generates a document ID string for every row by concatenating all key fields in the order defined in the query using a key delimiter. A custom representation of null literals for key fields can be defined. + +Attention A JSON format defines how to encode documents for the external system, therefore, it must be added as a [dependency](connect.html#formats). + +{% top %} + +Table Formats +------------- + +Flink provides a set of table formats that can be used with table connectors. + +A format tag indicates the format type for matching with a connector. + +### CSV Format + +The CSV format allows to read and write comma-separated rows. + +
    +
    +{% highlight java %} +.withFormat( + new Csv() + .field("field1", Types.STRING) // required: ordered format fields + .field("field2", Types.TIMESTAMP) + .fieldDelimiter(",") // optional: string delimiter "," by default + .lineDelimiter("\n") // optional: string delimiter "\n" by default + .quoteCharacter('"') // optional: single character for string values, empty by default + .commentPrefix('#') // optional: string to indicate comments, empty by default + .ignoreFirstLine() // optional: ignore the first line, by default it is not skipped + .ignoreParseErrors() // optional: skip records with parse error instead of failing by default +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +format: + type: csv + fields: # required: ordered format fields + - name: field1 + type: VARCHAR + - name: field2 + type: TIMESTAMP + field-delimiter: "," # optional: string delimiter "," by default + line-delimiter: "\n" # optional: string delimiter "\n" by default + quote-character: '"' # optional: single character for string values, empty by default + comment-prefix: '#' # optional: string to indicate comments, empty by default + ignore-first-line: false # optional: boolean flag to ignore the first line, by default it is not skipped + ignore-parse-errors: true # optional: skip records with parse error instead of failing by default +{% endhighlight %} +
    +
    + +The CSV format is included in Flink and does not require additional dependencies. + +Attention The CSV format for writing rows is limited at the moment. Only a custom field delimiter is supported as optional parameter. + +### JSON Format + +Format: Serialization Schema +Format: Deserialization Schema + +The JSON format allows to read and write JSON data that corresponds to a given format schema. The format schema can be defined either as a Flink type, as a JSON schema, or derived from the desired table schema. A Flink type enables a more SQL-like definition and mapping to the corresponding SQL data types. The JSON schema allows for more complex and nested structures. + +If the format schema is equal to the table schema, the schema can also be automatically derived. This allows for defining schema information only once. The names, types, and field order of the format are determined by the table's schema. Time attributes are ignored if their origin is not a field. A `from` definition in the table schema is interpreted as a field renaming in the format. + +
    +
    +{% highlight java %} +.withFormat( + new Json() + .failOnMissingField(true) // optional: flag whether to fail if a field is missing or not, false by default + + // required: define the schema either by using type information which parses numbers to corresponding types + .schema(Type.ROW(...)) + + // or by using a JSON schema which parses to DECIMAL and TIMESTAMP + .jsonSchema( + "{" + + " type: 'object'," + + " properties: {" + + " lon: {" + + " type: 'number'" + + " }," + + " rideTime: {" + + " type: 'string'," + + " format: 'date-time'" + + " }" + + " }" + + "}" + ) + + // or use the table's schema + .deriveSchema() +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +format: + type: json + fail-on-missing-field: true # optional: flag whether to fail if a field is missing or not, false by default + + # required: define the schema either by using a type string which parses numbers to corresponding types + schema: "ROW(lon FLOAT, rideTime TIMESTAMP)" + + # or by using a JSON schema which parses to DECIMAL and TIMESTAMP + json-schema: > + { + type: 'object', + properties: { + lon: { + type: 'number' + }, + rideTime: { + type: 'string', + format: 'date-time' + } + } + } + + # or use the table's schema + derive-schema: true +{% endhighlight %} +
    +
    + +The following table shows the mapping of JSON schema types to Flink SQL types: + +| JSON schema | Flink SQL | +| :-------------------------------- | :---------------------- | +| `object` | `ROW` | +| `boolean` | `BOOLEAN` | +| `array` | `ARRAY[_]` | +| `number` | `DECIMAL` | +| `integer` | `DECIMAL` | +| `string` | `VARCHAR` | +| `string` with `format: date-time` | `TIMESTAMP` | +| `string` with `format: date` | `DATE` | +| `string` with `format: time` | `TIME` | +| `string` with `encoding: base64` | `ARRAY[TINYINT]` | +| `null` | `NULL` (unsupported yet)| + + +Currently, Flink supports only a subset of the [JSON schema specification](http://json-schema.org/) `draft-07`. Union types (as well as `allOf`, `anyOf`, `not`) are not supported yet. `oneOf` and arrays of types are only supported for specifying nullability. + +Simple references that link to a common definition in the document are supported as shown in the more complex example below: + +{% highlight json %} +{ + "definitions": { + "address": { + "type": "object", + "properties": { + "street_address": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + } + }, + "required": [ + "street_address", + "city", + "state" + ] + } + }, + "type": "object", + "properties": { + "billing_address": { + "$ref": "#/definitions/address" + }, + "shipping_address": { + "$ref": "#/definitions/address" + }, + "optional_address": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/definitions/address" + } + ] + } + } +} +{% endhighlight %} + +**Missing Field Handling:** By default, a missing JSON field is set to `null`. You can enable strict JSON parsing that will cancel the source (and query) if a field is missing. + +Make sure to add the JSON format as a dependency. + + +### Apache Avro Format + +Format: Serialization Schema +Format: Deserialization Schema + +The [Apache Avro](https://avro.apache.org/) format allows to read and write Avro data that corresponds to a given format schema. The format schema can be defined either as a fully qualified class name of an Avro specific record or as an Avro schema string. If a class name is used, the class must be available in the classpath during runtime. + +
    +
    +{% highlight java %} +.withFormat( + new Avro() + + // required: define the schema either by using an Avro specific record class + .recordClass(User.class) + + // or by using an Avro schema + .avroSchema( + "{" + + " \"type\": \"record\"," + + " \"name\": \"test\"," + + " \"fields\" : [" + + " {\"name\": \"a\", \"type\": \"long\"}," + + " {\"name\": \"b\", \"type\": \"string\"}" + + " ]" + + "}" + ) +) +{% endhighlight %} +
    + +
    +{% highlight yaml %} +format: + type: avro + + # required: define the schema either by using an Avro specific record class + record-class: "org.organization.types.User" + + # or by using an Avro schema + avro-schema: > + { + "type": "record", + "name": "test", + "fields" : [ + {"name": "a", "type": "long"}, + {"name": "b", "type": "string"} + ] + } +{% endhighlight %} +
    +
    + +Avro types are mapped to the corresponding SQL data types. Union types are only supported for specifying nullability otherwise they are converted to an `ANY` type. The following table shows the mapping: + +| Avro schema | Flink SQL | +| :------------------------------------------ | :---------------------- | +| `record` | `ROW` | +| `enum` | `VARCHAR` | +| `array` | `ARRAY[_]` | +| `map` | `MAP[VARCHAR, _]` | +| `union` | non-null type or `ANY` | +| `fixed` | `ARRAY[TINYINT]` | +| `string` | `VARCHAR` | +| `bytes` | `ARRAY[TINYINT]` | +| `int` | `INT` | +| `long` | `BIGINT` | +| `float` | `FLOAT` | +| `double` | `DOUBLE` | +| `boolean` | `BOOLEAN` | +| `int` with `logicalType: date` | `DATE` | +| `int` with `logicalType: time-millis` | `TIME` | +| `int` with `logicalType: time-micros` | `INT` | +| `long` with `logicalType: timestamp-millis` | `TIMESTAMP` | +| `long` with `logicalType: timestamp-micros` | `BIGINT` | +| `bytes` with `logicalType: decimal` | `DECIMAL` | +| `fixed` with `logicalType: decimal` | `DECIMAL` | +| `null` | `NULL` (unsupported yet)| + +Avro uses [Joda-Time](http://www.joda.org/joda-time/) for representing logical date and time types in specific record classes. The Joda-Time dependency is not part of Flink's distribution. Therefore, make sure that Joda-Time is in your classpath together with your specific record class during runtime. Avro formats specified via a schema string do not require Joda-Time to be present. + +Make sure to add the Apache Avro dependency. + +{% top %} + +Further TableSources and TableSinks +----------------------------------- + +The following table sources and sinks have not yet been migrated (or have not been migrated entirely) to the new unified interfaces. + +These are the additional `TableSource`s which are provided with Flink: + +| **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description** +| `OrcTableSource` | `flink-orc` | Y | N | A `TableSource` for ORC files. + +These are the additional `TableSink`s which are provided with Flink: + +| **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description** +| `CsvTableSink` | `flink-table` | Y | Append | A simple sink for CSV files. +| `JDBCAppendTableSink` | `flink-jdbc` | Y | Append | Writes a Table to a JDBC table. +| `CassandraAppendTableSink` | `flink-connector-cassandra` | N | Append | Writes a Table to a Cassandra table. + +### OrcTableSource + +The `OrcTableSource` reads [ORC files](https://orc.apache.org). ORC is a file format for structured data and stores the data in a compressed, columnar representation. ORC is very storage efficient and supports projection and filter push-down. + +An `OrcTableSource` is created as shown below: + +
    +
    +{% highlight java %} + +// create Hadoop Configuration +Configuration config = new Configuration(); + +OrcTableSource orcTableSource = OrcTableSource.builder() + // path to ORC file(s). NOTE: By default, directories are recursively scanned. + .path("file:///path/to/data") + // schema of ORC files + .forOrcSchema("struct>>") + // Hadoop configuration + .withConfiguration(config) + // build OrcTableSource + .build(); +{% endhighlight %} +
    + +
    +{% highlight scala %} + +// create Hadoop Configuration +val config = new Configuration() + +val orcTableSource = OrcTableSource.builder() + // path to ORC file(s). NOTE: By default, directories are recursively scanned. + .path("file:///path/to/data") + // schema of ORC files + .forOrcSchema("struct>>") + // Hadoop configuration + .withConfiguration(config) + // build OrcTableSource + .build() +{% endhighlight %} +
    +
    + +**Note:** The `OrcTableSource` does not support ORC's `Union` type yet. + +{% top %} + +### CsvTableSink + +The `CsvTableSink` emits a `Table` to one or more CSV files. + +The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming/dynamic_tables.html#table-to-stream-conversion) for details. When emitting a streaming table, rows are written at least once (if checkpointing is enabled) and the `CsvTableSink` does not split output files into bucket files but continuously writes to the same files. + +
    +
    +{% highlight java %} + +CsvTableSink sink = new CsvTableSink( + path, // output path + "|", // optional: delimit files by '|' + 1, // optional: write to a single file + WriteMode.OVERWRITE); // optional: override existing files + +tableEnv.registerTableSink( + "csvOutputTable", + // specify table schema + new String[]{"f0", "f1"}, + new TypeInformation[]{Types.STRING, Types.INT}, + sink); + +Table table = ... +table.insertInto("csvOutputTable"); +{% endhighlight %} +
    + +
    +{% highlight scala %} + +val sink: CsvTableSink = new CsvTableSink( + path, // output path + fieldDelim = "|", // optional: delimit files by '|' + numFiles = 1, // optional: write to a single file + writeMode = WriteMode.OVERWRITE) // optional: override existing files + +tableEnv.registerTableSink( + "csvOutputTable", + // specify table schema + Array[String]("f0", "f1"), + Array[TypeInformation[_]](Types.STRING, Types.INT), + sink) + +val table: Table = ??? +table.insertInto("csvOutputTable") +{% endhighlight %} +
    +
    + +### JDBCAppendTableSink + +The `JDBCAppendTableSink` emits a `Table` to a JDBC connection. The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming/dynamic_tables.html#table-to-stream-conversion) for details. + +The `JDBCAppendTableSink` inserts each `Table` row at least once into the database table (if checkpointing is enabled). However, you can specify the insertion query using REPLACE or INSERT OVERWRITE to perform upsert writes to the database. + +To use the JDBC sink, you have to add the JDBC connector dependency (flink-jdbc) to your project. Then you can create the sink using JDBCAppendSinkBuilder: + +
    +
    +{% highlight java %} + +JDBCAppendTableSink sink = JDBCAppendTableSink.builder() + .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") + .setDBUrl("jdbc:derby:memory:ebookshop") + .setQuery("INSERT INTO books (id) VALUES (?)") + .setParameterTypes(INT_TYPE_INFO) + .build(); + +tableEnv.registerTableSink( + "jdbcOutputTable", + // specify table schema + new String[]{"id"}, + new TypeInformation[]{Types.INT}, + sink); + +Table table = ... +table.insertInto("jdbcOutputTable"); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val sink: JDBCAppendTableSink = JDBCAppendTableSink.builder() + .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") + .setDBUrl("jdbc:derby:memory:ebookshop") + .setQuery("INSERT INTO books (id) VALUES (?)") + .setParameterTypes(INT_TYPE_INFO) + .build() + +tableEnv.registerTableSink( + "jdbcOutputTable", + // specify table schema + Array[String]("id"), + Array[TypeInformation[_]](Types.INT), + sink) + +val table: Table = ??? +table.insertInto("jdbcOutputTable") +{% endhighlight %} +
    +
    + +Similar to using JDBCOutputFormat, you have to explicitly specify the name of the JDBC driver, the JDBC URL, the query to be executed, and the field types of the JDBC table. + +{% top %} + +### CassandraAppendTableSink + +The `CassandraAppendTableSink` emits a `Table` to a Cassandra table. The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming/dynamic_tables.html#table-to-stream-conversion) for details. + +The `CassandraAppendTableSink` inserts all rows at least once into the Cassandra table if checkpointing is enabled. However, you can specify the query as upsert query. + +To use the `CassandraAppendTableSink`, you have to add the Cassandra connector dependency (flink-connector-cassandra) to your project. The example below shows how to use the `CassandraAppendTableSink`. + +
    +
    +{% highlight java %} + +ClusterBuilder builder = ... // configure Cassandra cluster connection + +CassandraAppendTableSink sink = new CassandraAppendTableSink( + builder, + // the query must match the schema of the table + "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)"); + +tableEnv.registerTableSink( + "cassandraOutputTable", + // specify table schema + new String[]{"id", "name", "value"}, + new TypeInformation[]{Types.INT, Types.STRING, Types.DOUBLE}, + sink); + +Table table = ... +table.insertInto(cassandraOutputTable); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val builder: ClusterBuilder = ... // configure Cassandra cluster connection + +val sink: CassandraAppendTableSink = new CassandraAppendTableSink( + builder, + // the query must match the schema of the table + "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)") + +tableEnv.registerTableSink( + "cassandraOutputTable", + // specify table schema + Array[String]("id", "name", "value"), + Array[TypeInformation[_]](Types.INT, Types.STRING, Types.DOUBLE), + sink) + +val table: Table = ??? +table.insertInto(cassandraOutputTable) +{% endhighlight %} +
    +
    + +{% top %} diff --git a/docs/dev/table/functions.md b/docs/dev/table/functions.md new file mode 100644 index 00000000000000..f41139d14957c7 --- /dev/null +++ b/docs/dev/table/functions.md @@ -0,0 +1,5776 @@ +--- +title: "Built-In Functions" +nav-parent_id: tableapi +nav-pos: 31 +--- + + +Flink Table API & SQL provides users with a set of built-in functions for data transformations. This page gives a brief overview of them. +If a function that you need is not supported yet, you can implement a user-defined function. +If you think that the function is general enough, please open a Jira issue for it with a detailed description. + +* This will be replaced by the TOC +{:toc} + +Scalar Functions +---------------- + +The scalar functions take zero, one or more values as the input and return a single value as the result. + +### Comparison Functions + +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Comparison functionsDescription
    + {% highlight text %} +value1 = value2 +{% endhighlight %} + +

    Returns TRUE if value1 is equal to value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value1 <> value2 +{% endhighlight %} + +

    Returns TRUE if value1 is not equal to value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value1 > value2 +{% endhighlight %} + +

    Returns TRUE if value1 is greater than value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value1 >= value2 +{% endhighlight %} + +

    Returns TRUE if value1 is greater than or equal to value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value1 < value2 +{% endhighlight %} + +

    Returns TRUE if value1 is less than value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value1 <= value2 +{% endhighlight %} + +

    Returns TRUE if value1 is less than or equal to value2; returns UNKNOWN if value1 or value2 is NULL.

    +
    + {% highlight text %} +value IS NULL +{% endhighlight %} + +

    Returns TRUE if value is NULL.

    +
    + {% highlight text %} +value IS NOT NULL +{% endhighlight %} + +

    Returns TRUE if value is not NULL.

    +
    + {% highlight text %} +value1 IS DISTINCT FROM value2 +{% endhighlight %} + +

    Returns TRUE if two values are not equal. NULL values are treated as identical here.

    +

    E.g., 1 IS DISTINCT FROM NULL returns TRUE; + NULL IS DISTINCT FROM NULL returns FALSE.

    +
    + {% highlight text %} +value1 IS NOT DISTINCT FROM value2 +{% endhighlight %} + +

    Returns TRUE if two values are equal. NULL values are treated as identical here.

    +

    E.g., 1 IS NOT DISTINCT FROM NULL returns FALSE; + NULL IS NOT DISTINCT FROM NULL returns TRUE.

    +
    + {% highlight text %} +value1 BETWEEN [ ASYMMETRIC | SYMMETRIC ] value2 AND value3 +{% endhighlight %} + +

    By default (or with the ASYMMETRIC keyword), returns TRUE if value1 is greater than or equal to value2 and less than or equal to value3. + With the SYMMETRIC keyword, returns TRUE if value1 is inclusively between value2 and value3. + When either value2 or value3 is NULL, returns FALSE or UNKNOWN.

    +

    E.g., 12 BETWEEN 15 AND 12 returns FALSE; + 12 BETWEEN SYMMETRIC 15 AND 12 returns TRUE; + 12 BETWEEN 10 AND NULL returns UNKNOWN; + 12 BETWEEN NULL AND 10 returns FALSE; + 12 BETWEEN SYMMETRIC NULL AND 12 returns UNKNOWN.

    +
    + {% highlight text %} +value1 NOT BETWEEN [ ASYMMETRIC | SYMMETRIC ] value2 AND value3 +{% endhighlight %} + +

    By default (or with the ASYMMETRIC keyword), returns TRUE if value1 is less than value2 or greater than value3. + With the SYMMETRIC keyword, returns TRUE if value1 is not inclusively between value2 and value3. + When either value2 or value3 is NULL, returns TRUE or UNKNOWN.

    +

    E.g., 12 NOT BETWEEN 15 AND 12 returns TRUE; + 12 NOT BETWEEN SYMMETRIC 15 AND 12 returns FALSE; + 12 NOT BETWEEN NULL AND 15 returns UNKNOWN; + 12 NOT BETWEEN 15 AND NULL returns TRUE; + 12 NOT BETWEEN SYMMETRIC 12 AND NULL returns UNKNOWN.

    +
    + {% highlight text %} +string1 LIKE string2 [ ESCAPE char ] +{% endhighlight %} + +

    Returns TRUE if string1 matches pattern string2; returns UNKNOWN if string1 or string2 is NULL. An escape character can be defined if necessary.

    +

    Note: The escape character has not been supported yet.

    +
    + {% highlight text %} +string1 NOT LIKE string2 [ ESCAPE char ] +{% endhighlight %} + +

    Returns TRUE if string1 does not match pattern string2; returns UNKNOWN if string1 or string2 is NULL. An escape character can be defined if necessary.

    +

    Note: The escape character has not been supported yet.

    +
    + {% highlight text %} +string1 SIMILAR TO string2 [ ESCAPE char ] +{% endhighlight %} + +

    Returns TRUE if string1 matches SQL regular expression string2; returns UNKNOWN if string1 or string2 is NULL. An escape character can be defined if necessary.

    +

    Note: The escape character has not been supported yet.

    +
    + {% highlight text %} +string1 NOT SIMILAR TO string2 [ ESCAPE char ] +{% endhighlight %} + +

    Returns TRUE if string1 does not match SQL regular expression string2; returns UNKNOWN if string1 or string2 is NULL. An escape character can be defined if necessary.

    +

    Note: The escape character has not been supported yet.

    +
    + {% highlight text %} +value1 IN (value2 [, value3]* ) +{% endhighlight %} + +

    Returns TRUE if value1 exists in the given list (value2, value3, ...). + When (value2, value3, ...). contains NULL, returns TRUE if the element can be found and UNKNOWN otherwise. Always returns UNKNOWN if value1 is NULL.

    +

    E.g., 4 IN (1, 2, 3) returns FALSE; + 1 IN (1, 2, NULL) returns TRUE; + 4 IN (1, 2, NULL) returns UNKNOWN.

    +
    + {% highlight text %} +value1 NOT IN (value2 [, value3]* ) +{% endhighlight %} + +

    Returns TRUE if value1 does not exist in the given list (value2, value3, ...). + When (value2, value3, ...). contains NULL, returns FALSE if value1 can be found and UNKNOWN otherwise. Always returns UNKNOWN if value1 is NULL.

    +

    E.g., 4 NOT IN (1, 2, 3) returns TRUE; + 1 NOT IN (1, 2, NULL) returns FALSE; + 4 NOT IN (1, 2, NULL) returns UNKNOWN.

    +
    + {% highlight text %} +EXISTS (sub-query) +{% endhighlight %} + +

    Returns TRUE if sub-query returns at least one row. Only supported if the operation can be rewritten in a join and group operation.

    +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    +
    +{% highlight text %} +value IN (sub-query) +{% endhighlight %} + +

    Returns TRUE if value is equal to a row returned by sub-query.

    +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    +
    + {% highlight text %} +value NOT IN (sub-query) +{% endhighlight %} + +

    Returns TRUE if value is not equal to every row returned by sub-query.

    +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Comparison functionsDescription
    + {% highlight java %} +ANY1 === ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY1 !== ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is not equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY1 > ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY1 >= ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than or equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY1 < ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY1 <= ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than or equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight java %} +ANY.isNull +{% endhighlight %} + +

    Returns TRUE if ANY is NULL.

    +
    + {% highlight java %} +ANY.isNotNull +{% endhighlight %} + +

    Returns TRUE if ANY is not NULL.

    +
    + {% highlight java %} +STRING1.like(STRING2) +{% endhighlight %} + +

    Returns TRUE if STRING1 matches pattern STRING2; returns UNKNOWN if STRING1 or STRING2 is NULL.

    +

    E.g., "JoKn".like("Jo_n%") returns TRUE.

    +
    + {% highlight java %} +STRING.similar(STRING) +{% endhighlight %} + +

    Returns TRUE if STRING1 matches SQL regular expression STRING2; returns UNKNOWN if STRING1 or STRING2 is NULL.

    +

    E.g., "A".similar("A+") returns TRUE.

    +
    + {% highlight java %} +ANY1.in(ANY2, ANY3, ...) +{% endhighlight %} + +

    Returns TRUE if ANY1 exists in a given list (ANY2, ANY3, ...). + When (ANY2, ANY3, ...). contains NULL, returns TRUE if the element can be found and UNKNOWN otherwise. Always returns UNKNOWN if ANY1 is NULL.

    +

    E.g., 4.in(1, 2, 3) returns FALSE.

    +
    + {% highlight java %} +ANY.in(TABLE) +{% endhighlight %} + +

    Returns TRUE if ANY is equal to a row returned by sub-query TABLE.

    +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    +
    + {% highlight java %} +ANY1.between(ANY2, ANY3) +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than or equal to ANY2 and less than or equal to ANY3. + When either ANY2 or ANY3 is NULL, returns FALSE or UNKNOWN.

    +

    E.g., 12.between(15, 12) returns FALSE; + 12.between(10, Null(INT)) returns UNKNOWN; + 12.between(Null(INT), 10) returns FALSE.

    +
    + {% highlight java %} +ANY1.notBetween(ANY2, ANY3) +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than ANY2 or greater than ANY3. + When either ANY2 or ANY3 is NULL, returns TRUE or UNKNOWN.

    +

    E.g., 12.notBetween(15, 12) returns TRUE; + 12.notBetween(Null(INT), 15) returns UNKNOWN; + 12.notBetween(15, Null(INT)) returns TRUE.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Comparison functionsDescription
    + {% highlight scala %} +ANY1 === ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY1 !== ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is not equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY1 > ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY1 >= ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than or equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY1 < ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY1 <= ANY2 +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than or equal to ANY2; returns UNKNOWN if ANY1 or ANY2 is NULL.

    +
    + {% highlight scala %} +ANY.isNull +{% endhighlight %} + +

    Returns TRUE if ANY is NULL.

    +
    + {% highlight scala %} +ANY.isNotNull +{% endhighlight %} + +

    Returns TRUE if ANY is not NULL.

    +
    + {% highlight scala %} +STRING1.like(STRING2) +{% endhighlight %} + +

    Returns TRUE if STRING1 matches pattern STRING2; returns UNKNOWN if STRING1 or STRING2 is NULL.

    +

    E.g., "JoKn".like("Jo_n%") returns TRUE.

    +
    + {% highlight scala %} +STRING1.similar(STRING2) +{% endhighlight %} + +

    Returns TRUE if STRING1 matches SQL regular expression STRING2; returns UNKNOWN if STRING1 or STRING2 is NULL.

    +

    E.g., "A".similar("A+") returns TRUE.

    +
    + {% highlight scala %} +ANY1.in(ANY2, ANY3, ...) +{% endhighlight %} + +

    Returns TRUE if ANY1 exists in a given list (ANY2, ANY3, ...). + When (ANY2, ANY3, ...). contains NULL, returns TRUE if the element can be found and UNKNOWN otherwise. Always returns UNKNOWN if ANY1 is NULL.

    +

    E.g., 4.in(1, 2, 3) returns FALSE.

    +
    + {% highlight scala %} +ANY.in(TABLE) +{% endhighlight %} + +

    Returns TRUE if ANY is equal to a row returned by sub-query TABLE.

    +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    +
    + {% highlight scala %} +ANY1.between(ANY2, ANY3) +{% endhighlight %} + +

    Returns TRUE if ANY1 is greater than or equal to ANY2 and less than or equal to ANY3. + When either ANY2 or ANY3 is NULL, returns FALSE or UNKNOWN.

    +

    E.g., 12.between(15, 12) returns FALSE; + 12.between(10, Null(Types.INT)) returns UNKNOWN; + 12.between(Null(Types.INT), 10) returns FALSE.

    +
    + {% highlight scala %} +ANY1.notBetween(ANY2, ANY3) +{% endhighlight %} + +

    Returns TRUE if ANY1 is less than ANY2 or greater than ANY3. + When either ANY2 or ANY3 is NULL, returns TRUE or UNKNOWN.

    +

    E.g., 12.notBetween(15, 12) returns TRUE; + 12.notBetween(Null(Types.INT), 15) returns UNKNOWN; + 12.notBetween(15, Null(Types.INT)) returns TRUE.

    +
    +
    +
    + +{% top %} + +### Logical Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Logical functionsDescription
    + {% highlight text %} +boolean1 OR boolean2 +{% endhighlight %} + +

    Returns TRUE if boolean1 is TRUE or boolean2 is TRUE. Supports three-valued logic.

    +

    E.g., TRUE OR UNKNOWN returns TRUE.

    +
    + {% highlight text %} +boolean1 AND boolean2 +{% endhighlight %} + +

    Returns TRUE if boolean1 and boolean2 are both TRUE. Supports three-valued logic.

    +

    E.g., TRUE AND UNKNOWN returns UNKNOWN.

    +
    + {% highlight text %} +NOT boolean +{% endhighlight %} + +

    Returns TRUE if boolean is FALSE; returns FALSE if boolean is TRUE; returns UNKNOWN if boolean is UNKNOWN.

    +
    + {% highlight text %} +boolean IS FALSE +{% endhighlight %} + +

    Returns TRUE if boolean is FALSE; returns FALSE if boolean is TRUE or UNKNOWN.

    +
    + {% highlight text %} +boolean IS NOT FALSE +{% endhighlight %} + +

    Returns TRUE if boolean is TRUE or UNKNOWN; returns FALSE if boolean is FALSE.

    +
    + {% highlight text %} +boolean IS TRUE +{% endhighlight %} + +

    Returns TRUE if boolean is TRUE; returns FALSE if boolean is FALSE or UNKNOWN.

    +
    + {% highlight text %} +boolean IS NOT TRUE +{% endhighlight %} + +

    Returns TRUE if boolean is FALSE or UNKNOWN; returns FALSE if boolean is FALSE.

    +
    + {% highlight text %} +boolean IS UNKNOWN +{% endhighlight %} + +

    Returns TRUE if boolean is UNKNOWN; returns FALSE if boolean is TRUE or FALSE.

    +
    + {% highlight text %} +boolean IS NOT UNKNOWN +{% endhighlight %} + +

    Returns TRUE if boolean is TRUE or FALSE; returns FALSE if boolean is UNKNOWN.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Logical functionsDescription
    + {% highlight java %} +BOOLEAN1 || BOOLEAN2 +{% endhighlight %} + +

    Returns TRUE if BOOLEAN1 is TRUE or BOOLEAN2 is TRUE. Supports three-valued logic.

    +

    E.g., true || Null(BOOLEAN) returns TRUE.

    +
    + {% highlight java %} +BOOLEAN1 && BOOLEAN2 +{% endhighlight %} + +

    Returns TRUE if BOOLEAN1 and BOOLEAN2 are both TRUE. Supports three-valued logic.

    +

    E.g., true && Null(BOOLEAN) returns UNKNOWN.

    +
    + {% highlight java %} +!BOOLEAN +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE; returns FALSE if BOOLEAN is TRUE; returns UNKNOWN if BOOLEAN is UNKNOWN.

    +
    + {% highlight java %} +BOOLEAN.isTrue +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is TRUE; returns FALSE if BOOLEAN is FALSE or UNKNOWN.

    +
    + {% highlight java %} +BOOLEAN.isFalse +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE; returns FALSE if BOOLEAN is TRUE or UNKNOWN.

    +
    + {% highlight java %} +BOOLEAN.isNotTrue +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE or UNKNOWN; returns FALSE if BOOLEAN is FALSE.

    +
    + {% highlight java %} +BOOLEAN.isNotFalse +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is TRUE or UNKNOWN; returns FALSE if BOOLEAN is FALSE.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Logical functionsDescription
    + {% highlight scala %} +BOOLEAN1 || BOOLEAN2 +{% endhighlight %} + +

    Returns TRUE if BOOLEAN1 is TRUE or BOOLEAN2 is TRUE. Supports three-valued logic.

    +

    E.g., true || Null(Types.BOOLEAN) returns TRUE.

    +
    + {% highlight scala %} +BOOLEAN1 && BOOLEAN2 +{% endhighlight %} + +

    Returns TRUE if BOOLEAN1 and BOOLEAN2 are both TRUE. Supports three-valued logic.

    +

    E.g., true && Null(Types.BOOLEAN) returns UNKNOWN.

    +
    + {% highlight scala %} +!BOOLEAN +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE; returns FALSE if BOOLEAN is TRUE; returns UNKNOWN if BOOLEAN is UNKNOWN.

    +
    + {% highlight scala %} +BOOLEAN.isTrue +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is TRUE; returns FALSE if BOOLEAN is FALSE or UNKNOWN.

    +
    + {% highlight scala %} +BOOLEAN.isFalse +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE; returns FALSE if BOOLEAN is TRUE or UNKNOWN.

    +
    + {% highlight scala %} +BOOLEAN.isNotTrue +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is FALSE or UNKNOWN; returns FALSE if BOOLEAN is FALSE.

    +
    + {% highlight scala %} +BOOLEAN.isNotFalse +{% endhighlight %} + +

    Returns TRUE if BOOLEAN is TRUE or UNKNOWN; returns FALSE if BOOLEAN is FALSE.

    +
    +
    +
    + +{% top %} + +### Arithmetic Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Arithmetic functionsDescription
    + {% highlight text %} ++ numeric +{% endhighlight %} + +

    Returns numeric.

    +
    + {% highlight text %} +- numeric +{% endhighlight %} + +

    Returns negative numeric.

    +
    + {% highlight text %} +numeric1 + numeric2 +{% endhighlight %} + +

    Returns numeric1 plus numeric2.

    +
    + {% highlight text %} +numeric1 - numeric2 +{% endhighlight %} + +

    Returns numeric1 minus numeric2.

    +
    + {% highlight text %} +numeric1 * numeric2 +{% endhighlight %} + +

    Returns numeric1 multiplied by numeric2.

    +
    + {% highlight text %} +numeric1 / numeric2 +{% endhighlight %} + +

    Returns numeric1 divided by numeric2.

    +
    + {% highlight text %} +POWER(numeric1, numeric2) +{% endhighlight %} + +

    Returns numeric1 raised to the power of numeric2.

    +
    + {% highlight text %} +ABS(numeric) +{% endhighlight %} + +

    Returns the absolute value of numeric.

    +
    + {% highlight text %} +MOD(numeric1, numeric2) +{% endhighlight %} + +

    Returns the remainder (modulus) of numeric1 divided by numeric2. The result is negative only if numeric1 is negative.

    +
    + {% highlight text %} +SQRT(numeric) +{% endhighlight %} + +

    Returns the square root of numeric.

    +
    + {% highlight text %} +LN(numeric) +{% endhighlight %} + +

    Returns the natural logarithm (base e) of numeric.

    +
    + {% highlight text %} +LOG10(numeric) +{% endhighlight %} + +

    Returns the base 10 logarithm of numeric.

    +
    + {% highlight text %} +LOG2(numeric) +{% endhighlight %} + +

    Returns the base 2 logarithm of numeric.

    +
    + {% highlight text %} +LOG(numeric2) +LOG(numeric1, numeric2) +{% endhighlight %} + +

    When called with one argument, returns the natural logarithm of numeric2. When called with two arguments, this function returns the logarithm of numeric2 to the base numeric1.

    +

    Note: Currently, numeric2 must be greater than 0 and numeric1 must be greater than 1.

    +
    + {% highlight text %} +EXP(numeric) +{% endhighlight %} + +

    Returns e raised to the power of numeric.

    +
    + {% highlight text %} +CEIL(numeric) +CEILING(numeric) +{% endhighlight %} + +

    Rounds numeric up, and returns the smallest number that is greater than or equal to numeric.

    +
    + {% highlight text %} +FLOOR(numeric) +{% endhighlight %} + +

    Rounds numeric down, and returns the largest number that is less than or equal to numeric.

    +
    + {% highlight text %} +SIN(numeric) +{% endhighlight %} + +

    Returns the sine of numeric.

    +
    + {% highlight text %} +SINH(numeric) +{% endhighlight %} + +

    Returns the hyperbolic sine of numeric.

    +

    The return type is DOUBLE.

    +
    + {% highlight text %} +COS(numeric) +{% endhighlight %} + +

    Returns the cosine of numeric.

    +
    + {% highlight text %} +TAN(numeric) +{% endhighlight %} + +

    Returns the tangent of numeric.

    +
    + {% highlight text %} +TANH(numeric) +{% endhighlight %} + +

    Returns the hyperbolic tangent of numeric.

    +

    The return type is DOUBLE.

    +
    + {% highlight text %} +COT(numeric) +{% endhighlight %} + +

    Returns the cotangent of a numeric.

    +
    + {% highlight text %} +ASIN(numeric) +{% endhighlight %} + +

    Returns the arc sine of numeric.

    +
    + {% highlight text %} +ACOS(numeric) +{% endhighlight %} + +

    Returns the arc cosine of numeric.

    +
    + {% highlight text %} +ATAN(numeric) +{% endhighlight %} + +

    Returns the arc tangent of numeric.

    +
    + {% highlight text %} +ATAN2(numeric1, numeric2) +{% endhighlight %} + +

    Returns the arc tangent of a coordinate (numeric1, numeric2).

    +
    + {% highlight text %} +COSH(numeric) +{% endhighlight %} + +

    Returns the hyperbolic cosine of NUMERIC.

    +

    Return value type is DOUBLE.

    +
    + {% highlight text %} +DEGREES(numeric) +{% endhighlight %} + +

    Returns the degree representation of a radian numeric.

    +
    + {% highlight text %} +RADIANS(numeric) +{% endhighlight %} + +

    Returns the radian representation of a degree numeric.

    +
    + {% highlight text %} +SIGN(numeric) +{% endhighlight %} + +

    Returns the signum of numeric.

    +
    + {% highlight text %} +ROUND(numeric, integer) +{% endhighlight %} + +

    Returns a number rounded to integer decimal places for numeric.

    +
    + {% highlight text %} +PI +{% endhighlight %} + +

    Returns a value that is closer than any other values to pi.

    +
    + {% highlight text %} +E() +{% endhighlight %} + +

    Returns a value that is closer than any other values to e.

    +
    + {% highlight text %} +RAND() +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    +
    + {% highlight text %} +RAND(integer) +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with an initial seed integer. Two RAND functions will return identical sequences of numbers if they have the same initial seed.

    +
    + {% highlight text %} +RAND_INTEGER(integer) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and integer (exclusive).

    +
    + {% highlight text %} +RAND_INTEGER(integer1, integer2) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and the specified value (exclusive) with an initial seed. Two RAND_INTEGER functions will return identical sequences of numbers if they have the same initial seed and bound.

    +
    + {% highlight text %} +UUID() +{% endhighlight %} + +

    Returns an UUID (Universally Unique Identifier) string (e.g., "3d3c68f7-f608-473f-b60c-b0c44ad4cc4e") according to RFC 4122 type 4 (pseudo randomly generated) UUID. The UUID is generated using a cryptographically strong pseudo random number generator.

    +
    + {% highlight text %} +BIN(integer) +{% endhighlight %} + +

    Returns a string representation of integer in binary format. Returns NULL if integer is NULL.

    +

    E.g. BIN(4) returns '100' and BIN(12) returns '1100'.

    +
    +{% highlight text %} +HEX(numeric) +HEX(string) + {% endhighlight %} + +

    Returns a string representation of an integer numeric value or a string in hex format. Returns NULL if the argument is NULL.

    +

    E.g. a numeric 20 leads to "14", a numeric 100 leads to "64", a string "hello,world" leads to "68656C6C6F2C776F726C64".

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Arithmetic functionsDescription
    + {% highlight java %} ++ NUMERIC +{% endhighlight %} + +

    Returns NUMERIC.

    +
    + {% highlight java %} +- NUMERIC +{% endhighlight %} + +

    Returns negative NUMERIC.

    +
    + {% highlight java %} +NUMERIC1 + NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 plus NUMERIC2.

    +
    + {% highlight java %} +NUMERIC1 - NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 minus NUMERIC2.

    +
    + {% highlight java %} +NUMERIC1 * NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 multiplied by NUMERIC2.

    +
    + {% highlight java %} +NUMERIC1 / NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 divided by NUMERIC2.

    +
    + {% highlight java %} +NUMERIC1.power(NUMERIC2) +{% endhighlight %} + +

    Returns NUMERIC1 raised to the power of NUMERIC2.

    +
    + {% highlight java %} +NUMERIC.abs() +{% endhighlight %} + +

    Returns the absolute value of NUMERIC.

    +
    + {% highlight java %} +NUMERIC1 % NUMERIC2 +{% endhighlight %} + +

    Returns the remainder (modulus) of NUMERIC1 divided by NUMERIC2. The result is negative only if numeric1 is negative.

    +
    + {% highlight java %} +NUMERIC.sqrt() +{% endhighlight %} + +

    Returns the square root of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.ln() +{% endhighlight %} + +

    Returns the natural logarithm (base e) of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.log10() +{% endhighlight %} + +

    Returns the base 10 logarithm of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.log2() +{% endhighlight %} + +

    Returns the base 2 logarithm of NUMERIC.

    +
    + {% highlight java %} +NUMERIC1.log() +NUMERIC1.log(NUMERIC2) +{% endhighlight %} + +

    When called without argument, returns the natural logarithm of NUMERIC1. When called with an argument, returns the logarithm of NUMERIC1 to the base NUMERIC2.

    +

    Note: Currently, NUMERIC1 must be greater than 0 and NUMERIC2 must be greater than 1.

    +
    + {% highlight java %} +NUMERIC.exp() +{% endhighlight %} + +

    Returns e raised to the power of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.ceil() +{% endhighlight %} + +

    Rounds NUMERIC up, and returns the smallest number that is greater than or equal to NUMERIC.

    +
    + {% highlight java %} +NUMERIC.floor() +{% endhighlight %} + +

    Rounds NUMERIC down, and returns the largest number that is less than or equal to NUMERIC.

    +
    + {% highlight java %} +NUMERIC.sin() +{% endhighlight %} + +

    Returns the sine of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.sinh() +{% endhighlight %} + +

    Returns the hyperbolic sine of NUMERIC.

    +

    The return type is DOUBLE.

    +
    + {% highlight java %} +NUMERIC.cos() +{% endhighlight %} + +

    Returns the cosine of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.tan() +{% endhighlight %} + +

    Returns the tangent of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.tanh() +{% endhighlight %} + +

    Returns the hyperbolic tangent of NUMERIC.

    +

    The return type is DOUBLE.

    +
    + {% highlight java %} +NUMERIC.cot() +{% endhighlight %} + +

    Returns the cotangent of a NUMERIC.

    +
    + {% highlight java %} +NUMERIC.asin() +{% endhighlight %} + +

    Returns the arc sine of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.acos() +{% endhighlight %} + +

    Returns the arc cosine of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.atan() +{% endhighlight %} + +

    Returns the arc tangent of NUMERIC.

    +
    + {% highlight java %} +atan2(NUMERIC1, NUMERIC2) +{% endhighlight %} + +

    Returns the arc tangent of a coordinate (NUMERIC1, NUMERIC2).

    +
    + {% highlight java %} +NUMERIC.cosh() +{% endhighlight %} + +

    Returns the hyperbolic cosine of NUMERIC.

    +

    Return value type is DOUBLE.

    +
    + {% highlight java %} +NUMERIC.degrees() +{% endhighlight %} + +

    Returns the degree representation of a radian NUMERIC.

    +
    + {% highlight java %} +NUMERIC.radians() +{% endhighlight %} + +

    Returns the radian representation of a degree NUMERIC.

    +
    + {% highlight java %} +NUMERIC.sign() +{% endhighlight %} + +

    Returns the signum of NUMERIC.

    +
    + {% highlight java %} +NUMERIC.round(INT) +{% endhighlight %} + +

    Returns a number rounded to INT decimal places for NUMERIC.

    +
    + {% highlight java %} +pi() +{% endhighlight %} + +

    Returns a value that is closer than any other values to pi.

    +
    + {% highlight java %} +e() +{% endhighlight %} + +

    Returns a value that is closer than any other values to e.

    +
    + {% highlight java %} +rand() +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    +
    + {% highlight java %} +rand(INTEGER) +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with an initial seed INTEGER. Two RAND functions will return identical sequences of numbers if they have the same initial seed.

    +
    + {% highlight java %} +randInteger(INTEGER) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and INTEGER (exclusive).

    +
    + {% highlight java %} +randInteger(INTEGER1, INTEGER2) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and INTEGER2 (exclusive) with an initial seed INTEGER1. Two randInteger functions will return identical sequences of numbers if they have same initial seed and bound.

    +
    + {% highlight java %} +uuid() +{% endhighlight %} + +

    Returns an UUID (Universally Unique Identifier) string (e.g., "3d3c68f7-f608-473f-b60c-b0c44ad4cc4e") according to RFC 4122 type 4 (pseudo randomly generated) UUID. The UUID is generated using a cryptographically strong pseudo random number generator.

    +
    + {% highlight java %} +INTEGER.bin() +{% endhighlight %} + +

    Returns a string representation of INTEGER in binary format. Returns NULL if INTEGER is NULL.

    +

    E.g., 4.bin() returns "100" and 12.bin() returns "1100".

    +
    + {% highlight java %} +NUMERIC.hex() +STRING.hex() +{% endhighlight %} + +

    Returns a string representation of an integer NUMERIC value or a STRING in hex format. Returns NULL if the argument is NULL.

    +

    E.g. a numeric 20 leads to "14", a numeric 100 leads to "64", a string "hello,world" leads to "68656C6C6F2C776F726C64".

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Arithmetic functionsDescription
    + {% highlight scala %} ++ NUMERIC +{% endhighlight %} + +

    Returns NUMERIC.

    +
    + {% highlight scala %} +- NUMERIC +{% endhighlight %} + +

    Returns negative NUMERIC.

    +
    + {% highlight scala %} +NUMERIC1 + NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 plus NUMERIC2.

    +
    + {% highlight scala %} +NUMERIC1 - NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 minus NUMERIC2.

    +
    + {% highlight scala %} +NUMERIC1 * NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 multiplied by NUMERIC2.

    +
    + {% highlight scala %} +NUMERIC1 / NUMERIC2 +{% endhighlight %} + +

    Returns NUMERIC1 divided by NUMERIC2.

    +
    + {% highlight scala %} +NUMERIC1.power(NUMERIC2) +{% endhighlight %} + +

    Returns NUMERIC1 raised to the power of NUMERIC2.

    +
    + {% highlight scala %} +NUMERIC.abs() +{% endhighlight %} + +

    Returns the absolute value of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC1 % NUMERIC2 +{% endhighlight %} + +

    Returns the remainder (modulus) of NUMERIC1 divided by NUMERIC2. The result is negative only if numeric1 is negative.

    +
    + {% highlight scala %} +NUMERIC.sqrt() +{% endhighlight %} + +

    Returns the square root of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.ln() +{% endhighlight %} + +

    Returns the natural logarithm (base e) of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.log10() +{% endhighlight %} + +

    Returns the base 10 logarithm of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.log2() +{% endhighlight %} + +

    Returns the base 2 logarithm of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC1.log() +NUMERIC1.log(NUMERIC2) +{% endhighlight %} + +

    When called without argument, returns the natural logarithm of NUMERIC1. When called with an argument, returns the logarithm of NUMERIC1 to the base NUMERIC2.

    +

    Note: Currently, NUMERIC1 must be greater than 0 and NUMERIC2 must be greater than 1.

    +
    + {% highlight scala %} +NUMERIC.exp() +{% endhighlight %} + +

    Returns e raised to the power of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.ceil() +{% endhighlight %} + +

    Rounds NUMERIC up, and returns the smallest number that is greater than or equal to NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.floor() +{% endhighlight %} + +

    Rounds NUMERIC down, and returns the largest number that is less than or equal to NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.sin() +{% endhighlight %} + +

    Returns the sine of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.sinh() +{% endhighlight %} + +

    Returns the hyperbolic sine of NUMERIC.

    +

    The return type is DOUBLE.

    +
    + {% highlight scala %} +NUMERIC.cos() +{% endhighlight %} + +

    Returns the cosine of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.tan() +{% endhighlight %} + +

    Returns the tangent of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.tanh() +{% endhighlight %} + +

    Returns the hyperbolic tangent of NUMERIC.

    +

    The return type is DOUBLE.

    +
    + {% highlight scala %} +NUMERIC.cot() +{% endhighlight %} + +

    Returns the cotangent of a NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.asin() +{% endhighlight %} + +

    Returns the arc sine of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.acos() +{% endhighlight %} + +

    Returns the arc cosine of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.atan() +{% endhighlight %} + +

    Returns the arc tangent of NUMERIC.

    +
    + {% highlight scala %} +atan2(NUMERIC1, NUMERIC2) +{% endhighlight %} + +

    Returns the arc tangent of a coordinate (NUMERIC1, NUMERIC2).

    +
    + {% highlight scala %} +NUMERIC.cosh() +{% endhighlight %} + +

    Returns the hyperbolic cosine of NUMERIC.

    +

    Return value type is DOUBLE.

    +
    + {% highlight scala %} +NUMERIC.degrees() +{% endhighlight %} + +

    Returns the degree representation of a radian NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.radians() +{% endhighlight %} + +

    Returns the radian representation of a degree NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.sign() +{% endhighlight %} + +

    Returns the signum of NUMERIC.

    +
    + {% highlight scala %} +NUMERIC.round(INT) +{% endhighlight %} + +

    Returns a number rounded to INT decimal places for NUMERIC.

    +
    + {% highlight scala %} +pi() +{% endhighlight %} + +

    Returns a value that is closer than any other values to pi.

    +
    + {% highlight scala %} +e() +{% endhighlight %} + +

    Returns a value that is closer than any other values to e.

    +
    + {% highlight scala %} +rand() +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    +
    + {% highlight scala %} +rand(INTEGER) +{% endhighlight %} + +

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with an initial seed INTEGER. Two RAND functions will return identical sequences of numbers if they have the same initial seed.

    +
    + {% highlight scala %} +randInteger(INTEGER) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and INTEGER (exclusive).

    +
    + {% highlight scala %} +randInteger(INTEGER1, INTEGER2) +{% endhighlight %} + +

    Returns a pseudorandom integer value between 0 (inclusive) and INTEGER2 (exclusive) with an initial seed INTEGER1. Two randInteger functions will return identical sequences of numbers if they have same initial seed and bound.

    +
    + {% highlight scala %} +uuid() +{% endhighlight %} + +

    Returns an UUID (Universally Unique Identifier) string (e.g., "3d3c68f7-f608-473f-b60c-b0c44ad4cc4e") according to RFC 4122 type 4 (pseudo randomly generated) UUID. The UUID is generated using a cryptographically strong pseudo random number generator.

    +
    + {% highlight scala %} +INTEGER.bin() +{% endhighlight %} + +

    Returns a string representation of INTEGER in binary format. Returns NULL if INTEGER is NULL.

    +

    E.g., 4.bin() returns "100" and 12.bin() returns "1100".

    +
    + {% highlight scala %} +NUMERIC.hex() +STRING.hex() +{% endhighlight %} + +

    Returns a string representation of an integer NUMERIC value or a STRING in hex format. Returns NULL if the argument is NULL.

    +

    E.g. a numeric 20 leads to "14", a numeric 100 leads to "64", a string "hello,world" leads to "68656C6C6F2C776F726C64".

    +
    +
    +
    + +{% top %} + +### String Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    String functionsDescription
    + {% highlight text %} +string1 || string2 +{% endhighlight %} + +

    Returns the concatenation of string1 and string2.

    +
    + {% highlight text %} +CHAR_LENGTH(string) +CHARACTER_LENGTH(string) +{% endhighlight %} + +

    Returns the number of characters in string.

    +
    + {% highlight text %} +UPPER(string) +{% endhighlight %} + +

    Returns string in uppercase.

    +
    + {% highlight text %} +LOWER(string) +{% endhighlight %} + +

    Returns string in lowercase.

    +
    + {% highlight text %} +POSITION(string1 IN string2) +{% endhighlight %} + +

    Returns the position (start from 1) of the first occurrence of string1 in string2; returns 0 if string1 cannot be found in string2.

    +
    + {% highlight text %} +TRIM([ BOTH | LEADING | TRAILING ] string1 FROM string2) +{% endhighlight %} + +

    Returns a string that removes leading and/or trailing characters string1 from string2. By default, whitespaces at both sides are removed.

    +
    + {% highlight text %} +LTRIM(string) +{% endhighlight %} + +

    Returns a string that removes the left whitespaces from string.

    +

    E.g., LTRIM(' This is a test String.') returns "This is a test String.".

    +
    + {% highlight text %} +RTRIM(string) +{% endhighlight %} + +

    Returns a string that removes the right whitespaces from string.

    +

    E.g., RTRIM('This is a test String. ') returns "This is a test String.".

    +
    + {% highlight text %} +REPEAT(string, integer) +{% endhighlight %} + +

    Returns a string that repeats the base string integer times.

    +

    E.g., REPEAT('This is a test String.', 2) returns "This is a test String.This is a test String.".

    +
    + {% highlight text %} +REGEXP_REPLACE(string1, string2, string3) +{% endhighlight %} + +

    Returns a string from string1 with all the substrings that match a regular expression string2 consecutively being replaced with string3.

    +

    E.g., REGEXP_REPLACE('foobar', 'oo|ar', '') returns "fb".

    +
    + {% highlight text %} +OVERLAY(string1 PLACING string2 FROM integer1 [ FOR integer2 ]) +{% endhighlight %} + +

    Returns a string that replaces integer2 (string2's length by default) characters of string1 with string2 from position integer1.

    +

    E.g., OVERLAY('This is an old string' PLACING ' new' FROM 10 FOR 5) returns "This is a new string"

    +
    + {% highlight text %} +SUBSTRING(string FROM integer1 [ FOR integer2 ]) +{% endhighlight %} + +

    Returns a substring of string starting from position integer1 with length integer2 (to the end by default).

    +
    + {% highlight text %} +REPLACE(string1, string2, string3) +{% endhighlight %} + +

    Returns a new string which replaces all the occurrences of string2 with string3 (non-overlapping) from string1

    +

    E.g., REPLACE("hello world", "world", "flink") returns "hello flink"; REPLACE("ababab", "abab", "z") returns "zab".

    +
    + {% highlight text %} +REGEXP_EXTRACT(string1, string2[, integer]) +{% endhighlight %} + +

    Returns a string from string1 which extracted with a specified regular expression string2 and a regex match group index integer.

    +

    Note: The regex match group index starts from 1 and 0 means matching the whole regex. In addition, the regex match group index should not exceed the number of the defined groups.

    +

    E.g. REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2)" returns "bar".

    +
    + {% highlight text %} +INITCAP(string) +{% endhighlight %} + +

    Returns a new form of string with the first character of each word converted to uppercase and the rest characters to lowercase. Here a word means a sequences of alphanumeric characters.

    +
    + {% highlight text %} +CONCAT(string1, string2,...) +{% endhighlight %} + +

    Returns a string that concatenates string1, string2, .... Returns NULL if any argument is NULL.

    +

    E.g., CONCAT('AA', 'BB', 'CC') returns "AABBCC".

    +
    + {% highlight text %} +CONCAT_WS(string1, string2, string3,...) +{% endhighlight %} + +

    Returns a string that concatenates string2, string3, ... with a separator string1. The separator is added between the strings to be concatenated. Returns NULL If string1 is NULL. Compared with CONCAT(), CONCAT_WS() automatically skips NULL arguments.

    +

    E.g., CONCAT_WS('~', 'AA', NULL, 'BB', '', 'CC') returns "AA~BB~~CC".

    +
    + {% highlight text %} +LPAD(string1, integer, string2) +{% endhighlight %} + +

    Returns a new string from string1 left-padded with string2 to a length of integer characters. If the length of string1 is shorter than integer, returns string1 shortened to integer characters.

    +

    E.g., LPAD('hi',4,'??') returns "??hi"; LPAD('hi',1,'??') returns "h".

    +
    + {% highlight text %} +RPAD(string1, integer, string2) +{% endhighlight %} + +

    Returns a new string from string1 right-padded with string2 to a length of integer characters. If the length of string1 is shorter than integer, returns string1 shortened to integer characters.

    +

    E.g., RPAD('hi',4,'??') returns "hi??", RPAD('hi',1,'??') returns "h".

    +
    + {% highlight text %} +FROM_BASE64(string) +{% endhighlight %} + +

    Returns the base64-decoded result from string; returns NULL if string is NULL.

    +

    E.g., FROM_BASE64('aGVsbG8gd29ybGQ=') returns "hello world".

    +
    + {% highlight text %} +TO_BASE64(string) +{% endhighlight %} + +

    Returns the base64-encoded result from string; returns NULL if string is NULL.

    +

    E.g., TO_BASE64('hello world') returns "aGVsbG8gd29ybGQ=".

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    String functionsDescription
    + {% highlight java %} +STRING1 + STRING2 +{% endhighlight %} + +

    Returns the concatenation of STRING1 and STRING2.

    +
    + {% highlight java %} +STRING.charLength() +{% endhighlight %} + +

    Returns the number of characters in STRING.

    +
    + {% highlight java %} +STRING.upperCase() +{% endhighlight %} + +

    Returns STRING in uppercase.

    +
    + {% highlight java %} +STRING.lowerCase() +{% endhighlight %} + +

    Returns STRING in lowercase.

    +
    + {% highlight java %} +STRING1.position(STRING2) +{% endhighlight %} + +

    Returns the position (start from 1) of the first occurrence of STRING1 in STRING2; returns 0 if STRING1 cannot be found in STRING2.

    +
    + {% highlight java %} +STRING1.trim(LEADING, STRING2) +STRING1.trim(TRAILING, STRING2) +STRING1.trim(BOTH, STRING2) +STRING1.trim(BOTH) +STRING1.trim() +{% endhighlight %} + +

    Returns a string that removes leading and/or trailing characters STRING2 from STRING1. By default, whitespaces at both sides are removed.

    +
    + {% highlight java %} +STRING.ltrim() +{% endhighlight %} + +

    Returns a string that removes the left whitespaces from STRING.

    +

    E.g., ' This is a test String.'.ltrim() returns "This is a test String.".

    +
    + {% highlight java %} +STRING.rtrim() +{% endhighlight %} + +

    Returns a string that removes the right whitespaces from STRING.

    +

    E.g., 'This is a test String. '.rtrim() returns "This is a test String.".

    +
    + {% highlight java %} +STRING.repeat(INT) +{% endhighlight %} + +

    Returns a string that repeats the base STRING INT times.

    +

    E.g., 'This is a test String.'.repeat(2) returns "This is a test String.This is a test String.".

    +
    + {% highlight java %} +STRING1.regexpReplace(STRING2, STRING3) +{% endhighlight %} + +

    Returns a string from STRING1 with all the substrings that match a regular expression STRING2 consecutively being replaced with STRING3.

    +

    E.g., 'foobar'.regexpReplace('oo|ar', '') returns "fb".

    +
    + {% highlight java %} +STRING1.overlay(STRING2, INT1) +STRING1.overlay(STRING2, INT1, INT2) +{% endhighlight %} + +

    Returns a string that replaces INT2 (STRING2's length by default) characters of STRING1 with STRING2 from position INT1.

    +

    E.g., 'xxxxxtest'.overlay('xxxx', 6) returns "xxxxxxxxx"; 'xxxxxtest'.overlay('xxxx', 6, 2) returns "xxxxxxxxxst".

    +
    + {% highlight java %} +STRING.substring(INT1) +STRING.substring(INT1, INT2) +{% endhighlight %} + +

    Returns a substring of STRING starting from position INT1 with length INT2 (to the end by default).

    +
    + {% highlight java %} +STRING1.replace(STRING2, STRING3) +{% endhighlight %} + +

    Returns a new string which replaces all the occurrences of STRING2 with STRING3 (non-overlapping) from STRING1.

    +

    E.g., 'hello world'.replace('world', 'flink') returns 'hello flink'; 'ababab'.replace('abab', 'z') returns 'zab'.

    +
    + {% highlight java %} +STRING1.regexpExtract(STRING2[, INTEGER1]) +{% endhighlight %} + +

    Returns a string from STRING1 which extracted with a specified regular expression STRING2 and a regex match group index INTEGER1.

    +

    Note: The regex match group index starts from 1 and 0 means matching the whole regex. In addition, the regex match group index should not exceed the number of the defined groups.

    +

    E.g. 'foothebar'.regexpExtract('foo(.*?)(bar)', 2)" returns "bar".

    +
    + {% highlight java %} +STRING.initCap() +{% endhighlight %} + +

    Returns a new form of STRING with the first character of each word converted to uppercase and the rest characters to lowercase. Here a word means a sequences of alphanumeric characters.

    +
    + {% highlight java %} +concat(STRING1, STRING2, ...) +{% endhighlight %} + +

    Returns a string that concatenates STRING1, STRING2, .... Returns NULL if any argument is NULL.

    +

    E.g., concat('AA', 'BB', 'CC') returns "AABBCC".

    +
    + {% highlight java %} +concat_ws(STRING1, STRING2, STRING3, ...) +{% endhighlight %} + +

    Returns a string that concatenates STRING2, STRING3, ... with a separator STRING1. The separator is added between the strings to be concatenated. Returns NULL If STRING1 is NULL. Compared with concat(), concat_ws() automatically skips NULL arguments.

    +

    E.g., concat_ws('~', 'AA', Null(STRING), 'BB', '', 'CC') returns "AA~BB~~CC".

    +
    + {% highlight java %} +STRING1.lpad(INT, STRING2) +{% endhighlight %} + +

    Returns a new string from STRING1 left-padded with STRING2 to a length of INT characters. If the length of STRING1 is shorter than INT, returns STRING1 shortened to INT characters.

    +

    E.g., 'hi'.lpad(4, '??') returns "??hi"; 'hi'.lpad(1, '??') returns "h".

    +
    + {% highlight java %} +STRING1.rpad(INT, STRING2) +{% endhighlight %} + +

    Returns a new string from STRING1 right-padded with STRING2 to a length of INT characters. If the length of STRING1 is shorter than INT, returns STRING1 shortened to INT characters.

    +

    E.g., 'hi'.rpad(4, '??') returns "hi??"; 'hi'.rpad(1, '??') returns "h".

    +
    + {% highlight java %} +STRING.fromBase64() +{% endhighlight %} + +

    Returns the base64-decoded result from STRING; returns NULL if STRING is NULL.

    +

    E.g., 'aGVsbG8gd29ybGQ='.fromBase64() returns "hello world".

    +
    + {% highlight java %} +STRING.toBase64() +{% endhighlight %} + +

    Returns the base64-encoded result from STRING; returns NULL if STRING is NULL.

    +

    E.g., 'hello world'.toBase64() returns "aGVsbG8gd29ybGQ=".

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    String functionsDescription
    + {% highlight scala %} +STRING1 + STRING2 +{% endhighlight %} + +

    Returns the concatenation of STRING1 and STRING2.

    +
    + {% highlight scala %} +STRING.charLength() +{% endhighlight %} + +

    Returns the number of characters in STRING.

    +
    + {% highlight scala %} +STRING.upperCase() +{% endhighlight %} + +

    Returns STRING in uppercase.

    +
    + {% highlight scala %} +STRING.lowerCase() +{% endhighlight %} + +

    Returns STRING in lowercase.

    +
    + {% highlight scala %} +STRING1.position(STRING2) +{% endhighlight %} + +

    Returns the position (start from 1) of the first occurrence of STRING1 in STRING2; returns 0 if STRING1 cannot be found in STRING2.

    +
    + {% highlight scala %} +STRING.trim( + leading = true, + trailing = true, + character = " ") +{% endhighlight %} + +

    Returns a string that removes leading and/or trailing characters from STRING.

    +
    + {% highlight scala %} +STRING.ltrim() +{% endhighlight %} + +

    Returns a string that removes the left whitespaces from STRING.

    +

    E.g., " This is a test String.".ltrim() returns "This is a test String.".

    +
    + {% highlight scala %} +STRING.rtrim() +{% endhighlight %} + +

    Returns a string that removes the right whitespaces from STRING.

    +

    E.g., "This is a test String. ".rtrim() returns "This is a test String.".

    +
    + {% highlight scala %} +STRING.repeat(INT) +{% endhighlight %} + +

    Returns a string that repeats the base STRING INT times.

    +

    E.g., "This is a test String.".repeat(2) returns "This is a test String.This is a test String.".

    +
    + {% highlight scala %} +STRING1.regexpReplace(STRING2, STRING3) +{% endhighlight %} + +

    Returns a string from STRING1 with all the substrings that match a regular expression STRING2 consecutively being replaced with STRING3.

    +

    E.g. "foobar".regexpReplace("oo|ar", "") returns "fb".

    +
    + {% highlight scala %} +STRING1.overlay(STRING2, INT1) +STRING1.overlay(STRING2, INT1, INT2) +{% endhighlight %} + +

    Returns a string that replaces INT2 (STRING2's length by default) characters of STRING1 with STRING2 from position INT1.

    +

    E.g., "xxxxxtest".overlay("xxxx", 6) returns "xxxxxxxxx"; "xxxxxtest".overlay("xxxx", 6, 2) returns "xxxxxxxxxst".

    +
    + {% highlight scala %} +STRING.substring(INT1) +STRING.substring(INT1, INT2) +{% endhighlight %} + +

    Returns a substring of STRING starting from position INT1 with length INT2 (to the end by default).

    +
    + {% highlight scala %} +STRING1.replace(STRING2, STRING3) +{% endhighlight %} + +

    Returns a new string which replaces all the occurrences of STRING2 with STRING3 (non-overlapping) from STRING1.

    +

    E.g., "hello world".replace("world", "flink") returns "hello flink"; "ababab".replace("abab", "z") returns "zab".

    +
    + {% highlight scala %} +STRING1.regexpExtract(STRING2[, INTEGER1]) +{% endhighlight %} + +

    Returns a string from STRING1 which extracted with a specified regular expression STRING2 and a regex match group index INTEGER1.

    +

    Note: The regex match group index starts from 1 and 0 means matching the whole regex. In addition, the regex match group index should not exceed the number of the defined groups.

    +

    E.g. "foothebar".regexpExtract("foo(.*?)(bar)", 2)" returns "bar".

    +
    + {% highlight scala %} +STRING.initCap() +{% endhighlight %} + +

    Returns a new form of STRING with the first character of each word converted to uppercase and the rest characters to lowercase. Here a word means a sequences of alphanumeric characters.

    +
    + {% highlight scala %} +concat(STRING1, STRING2, ...) +{% endhighlight %} + +

    Returns a string that concatenates STRING1, STRING2, .... Returns NULL if any argument is NULL.

    +

    E.g., concat("AA", "BB", "CC") returns "AABBCC".

    +
    + {% highlight scala %} +concat_ws(STRING1, STRING2, STRING3, ...) +{% endhighlight %} + +

    Returns a string that concatenates STRING2, STRING3, ... with a separator STRING1. The separator is added between the strings to be concatenated. Returns NULL If STRING1 is NULL. Compared with concat(), concat_ws() automatically skips NULL arguments.

    +

    E.g., concat_ws("~", "AA", Null(Types.STRING), "BB", "", "CC") returns "AA~BB~~CC".

    +
    + {% highlight scala %} +STRING1.lpad(INT, STRING2) +{% endhighlight %} + +

    Returns a new string from STRING1 left-padded with STRING2 to a length of INT characters. If the length of STRING1 is shorter than INT, returns STRING1 shortened to INT characters.

    +

    E.g., "hi".lpad(4, "??") returns "??hi"; "hi".lpad(1, "??") returns "h".

    +
    + {% highlight scala %} +STRING1.rpad(INT, STRING2) +{% endhighlight %} + +

    Returns a new string from STRING1 right-padded with STRING2 to a length of INT characters. If the length of STRING1 is shorter than INT, returns STRING1 shortened to INT characters.

    +

    E.g., "hi".rpad(4, "??") returns "hi??"; "hi".rpad(1, "??") returns "h".

    +
    + {% highlight scala %} +STRING.fromBase64() +{% endhighlight %} + +

    Returns the base64-decoded result from STRING; returns null If STRING is NULL.

    +

    E.g., "aGVsbG8gd29ybGQ=".fromBase64() returns "hello world".

    +
    + {% highlight scala %} +STRING.toBase64() +{% endhighlight %} + +

    Returns the base64-encoded result from STRING; returns NULL if STRING is NULL.

    +

    E.g., "hello world".toBase64() returns "aGVsbG8gd29ybGQ=".

    +
    +
    +
    + +{% top %} + +### Temporal Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Temporal functionsDescription
    + {% highlight text %} +DATE string +{% endhighlight %} + +

    Returns a SQL date parsed from string in form of "yyyy-MM-dd".

    +
    + {% highlight text %} +TIME string +{% endhighlight %} + +

    Returns a SQL time parsed from string in form of "HH:mm:ss".

    +
    + {% highlight text %} +TIMESTAMP string +{% endhighlight %} + +

    Returns a SQL timestamp parsed from string in form of "yyyy-MM-dd HH:mm:ss[.SSS]".

    +
    + {% highlight text %} +INTERVAL string range +{% endhighlight %} + +

    Parses an interval string in the form "dd hh:mm:ss.fff" for SQL intervals of milliseconds or "yyyy-mm" for SQL intervals of months. An interval range might be DAY, MINUTE, DAY TO HOUR, or DAY TO SECOND for intervals of milliseconds; YEAR or YEAR TO MONTH for intervals of months.

    +

    E.g., INTERVAL '10 00:00:00.004' DAY TO SECOND, INTERVAL '10' DAY, or INTERVAL '2-10' YEAR TO MONTH return intervals.

    +
    + {% highlight text %} +CURRENT_DATE +{% endhighlight %} + +

    Returns the current SQL date in the UTC time zone.

    +
    + {% highlight text %} +CURRENT_TIME +{% endhighlight %} + +

    Returns the current SQL time in the UTC time zone.

    +
    + {% highlight text %} +CURRENT_TIMESTAMP +{% endhighlight %} + +

    Returns the current SQL timestamp in the UTC time zone.

    +
    + {% highlight text %} +LOCALTIME +{% endhighlight %} + +

    Returns the current SQL time in local time zone.

    +
    + {% highlight text %} +LOCALTIMESTAMP +{% endhighlight %} + +

    Returns the current SQL timestamp in local time zone.

    +
    + {% highlight text %} +EXTRACT(timeintervalunit FROM temporal) +{% endhighlight %} + +

    Returns a long value extracted from the timeintervalunit part of temporal.

    +

    E.g., EXTRACT(DAY FROM DATE '2006-06-05') returns 5.

    +
    + {% highlight text %} +YEAR(date) +{% endhighlight %} + +

    Returns the year from SQL date date. Equivalent to EXTRACT(YEAR FROM date).

    +

    E.g., YEAR(DATE '1994-09-27') returns 1994.

    +
    + {% highlight text %} +QUARTER(date) +{% endhighlight %} + +

    Returns the quarter of a year (an integer between 1 and 4) from SQL date date. Equivalent to EXTRACT(QUARTER FROM date).

    +

    E.g., QUARTER(DATE '1994-09-27') returns 3.

    +
    + {% highlight text %} +MONTH(date) +{% endhighlight %} + +

    Returns the month of a year (an integer between 1 and 12) from SQL date date. Equivalent to EXTRACT(MONTH FROM date).

    +

    E.g., MONTH(DATE '1994-09-27') returns 9.

    +
    + {% highlight text %} +WEEK(date) +{% endhighlight %} + +

    Returns the week of a year (an integer between 1 and 53) from SQL date date. Equivalent to EXTRACT(WEEK FROM date).

    +

    E.g., WEEK(DATE '1994-09-27') returns 39.

    +
    + {% highlight text %} +DAYOFYEAR(date) +{% endhighlight %} + +

    Returns the day of a year (an integer between 1 and 366) from SQL date date. Equivalent to EXTRACT(DOY FROM date).

    +

    E.g., DAYOFYEAR(DATE '1994-09-27') returns 270.

    +
    + {% highlight text %} +DAYOFMONTH(date) +{% endhighlight %} + +

    Returns the day of a month (an integer between 1 and 31) from SQL date date. Equivalent to EXTRACT(DAY FROM date).

    +

    E.g., DAYOFMONTH(DATE '1994-09-27') returns 27.

    +
    + {% highlight text %} +DAYOFWEEK(date) +{% endhighlight %} + +

    Returns the day of a week (an integer between 1 and 7; Sunday = 1) from SQL date date.Equivalent to EXTRACT(DOW FROM date).

    +

    E.g., DAYOFWEEK(DATE '1994-09-27') returns 3.

    +
    + {% highlight text %} +HOUR(timestamp) +{% endhighlight %} + +

    Returns the hour of a day (an integer between 0 and 23) from SQL timestamp timestamp. Equivalent to EXTRACT(HOUR FROM timestamp).

    +

    E.g., HOUR(TIMESTAMP '1994-09-27 13:14:15') returns 13.

    +
    + {% highlight text %} +MINUTE(timestamp) +{% endhighlight %} + +

    Returns the minute of an hour (an integer between 0 and 59) from SQL timestamp timestamp. Equivalent to EXTRACT(MINUTE FROM timestamp).

    +

    E.g., MINUTE(TIMESTAMP '1994-09-27 13:14:15') returns 14.

    +
    + {% highlight text %} +SECOND(timestamp) +{% endhighlight %} + +

    Returns the second of a minute (an integer between 0 and 59) from SQL timestamp. Equivalent to EXTRACT(SECOND FROM timestamp).

    +

    E.g., SECOND(TIMESTAMP '1994-09-27 13:14:15') returns 15.

    +
    + {% highlight text %} +FLOOR(timepoint TO timeintervalunit) +{% endhighlight %} + +

    Returns a value that rounds timepoint down to the time unit timeintervalunit.

    +

    E.g., FLOOR(TIME '12:44:31' TO MINUTE) returns 12:44:00.

    +
    + {% highlight text %} +CEIL(timepoint TO timeintervalunit) +{% endhighlight %} + +

    Returns a value that rounds timepoint up to the time unit timeintervalunit.

    +

    E.g., CEIL(TIME '12:44:31' TO MINUTE) returns 12:45:00.

    +
    + {% highlight text %} +(timepoint1, temporal1) OVERLAPS (timepoint2, temporal2) +{% endhighlight %} + +

    Returns TRUE if two time intervals defined by (timepoint1, temporal1) and (timepoint2, temporal2) overlap. The temporal values could be either a time point or a time interval.

    +

    E.g., (TIME '2:55:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:30:00', INTERVAL '2' HOUR) returns TRUE; (TIME '9:00:00', TIME '10:00:00') OVERLAPS (TIME '10:15:00', INTERVAL '3' HOUR) returns FALSE.

    +
    + {% highlight text %} +DATE_FORMAT(timestamp, string) +{% endhighlight %} + +

    Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use EXTRACT as a workaround.

    +
    + {% highlight text %} +TIMESTAMPADD(timeintervalunit, interval, timepoint) +{% endhighlight %} + +

    Returns a new time value that adds a (signed) integer interval to timepoint. The unit for interval is given by the unit argument, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR.

    +

    E.g., TIMESTAMPADD(WEEK, 1, DATE '2003-01-02') returns 2003-01-09.

    +
    + {% highlight text %} +TIMESTAMPDIFF(timepointunit, timepoint1, timepoint2) +{% endhighlight %} + +

    Returns the (signed) number of timepointunit between timepoint1 and timepoint2. The unit for the interval is given by the first argument, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR. See also the Time Interval and Point Unit Specifiers table.

    +

    E.g., TIMESTAMPDIFF(DAY, TIMESTAMP '2003-01-02 10:00:00', TIMESTAMP '2003-01-03 10:00:00') leads to 1.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Temporal functionsDescription
    + {% highlight java %} +STRING.toDate() +{% endhighlight %} + +

    Returns a SQL date parsed from STRING in form of "yyyy-MM-dd".

    +
    + {% highlight java %} +STRING.toTime() +{% endhighlight %} + +

    Returns a SQL time parsed from STRING in form of "HH:mm:ss".

    +
    + {% highlight java %} +STRING.toTimestamp() +{% endhighlight %} + +

    Returns a SQL timestamp parsed from STRING in form of "yyyy-MM-dd HH:mm:ss[.SSS]".

    +
    + {% highlight java %} +NUMERIC.year +NUMERIC.years +{% endhighlight %} + +

    Creates an interval of months for NUMERIC years.

    +
    + {% highlight java %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + +

    Creates an interval of months for NUMERIC quarters.

    +

    E.g., 2.quarters returns 6.

    +
    + {% highlight java %} +NUMERIC.month +NUMERIC.months +{% endhighlight %} + +

    Creates an interval of NUMERIC months.

    +
    + {% highlight java %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC weeks.

    +

    E.g., 2.weeks returns 1209600000.

    +
    + {% highlight java %} +NUMERIC.day +NUMERIC.days +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC days.

    +
    + {% highlight java %} +NUMERIC.hour +NUMERIC.hours +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC hours.

    +
    + {% highlight java %} +NUMERIC.minute +NUMERIC.minutes +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC minutes.

    +
    + {% highlight java %} +NUMERIC.second +NUMERIC.seconds +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC seconds.

    +
    + {% highlight java %} +NUMERIC.milli +NUMERIC.millis +{% endhighlight %} + +

    Creates an interval of NUMERIC milliseconds.

    +
    + {% highlight java %} +currentDate() +{% endhighlight %} + +

    Returns the current SQL date in the UTC time zone.

    +
    + {% highlight java %} +currentTime() +{% endhighlight %} + +

    Returns the current SQL time in the UTC time zone.

    +
    + {% highlight java %} +currentTimestamp() +{% endhighlight %} + +

    Returns the current SQL timestamp in the UTC time zone.

    +
    + {% highlight java %} +localTime() +{% endhighlight %} + +

    Returns the current SQL time in local time zone.

    +
    + {% highlight java %} +localTimestamp() +{% endhighlight %} + +

    Returns the current SQL timestamp in local time zone.

    +
    + {% highlight java %} +TEMPORAL.extract(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a long value extracted from the TIMEINTERVALUNIT part of temporal.

    +

    E.g., '2006-06-05'.toDate.extract(DAY) returns 5; '2006-06-05'.toDate.extract(QUARTER) returns 2.

    +
    + {% highlight java %} +TIMEPOINT.floor(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a value that rounds TIMEPOINT down to the time unit TIMEINTERVALUNIT.

    +

    E.g., '12:44:31'.toDate.floor(MINUTE) returns 12:44:00.

    +
    + {% highlight java %} +TIMEPOINT.ceil(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a value that rounds TIMEPOINT up to the time unit TIMEINTERVALUNIT.

    +

    E.g., '12:44:31'.toTime.floor(MINUTE) returns 12:45:00.

    +
    + {% highlight java %} +temporalOverlaps(TIMEPOINT1, TEMPORAL1, TIMEPOINT2, TEMPORAL2) +{% endhighlight %} + +

    Returns TRUE if two time intervals defined by (TIMEPOINT1, TEMPORAL1) and (TIMEPOINT2, TEMPORAL2) overlap. The temporal values could be either a time point or a time interval.

    +

    E.g., temporalOverlaps('2:55:00'.toTime, 1.hour, '3:30:00'.toTime, 2.hour) returns TRUE.

    +
    + {% highlight java %} +dateFormat(TIMESTAMP, STRING) +{% endhighlight %} + +

    Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.

    +
    + {% highlight java %} +timestampDiff(TIMEPOINTUNIT, TIMEPOINT1, TIMEPOINT2) +{% endhighlight %} + +

    Returns the (signed) number of TIMEPOINTUNIT between TIMEPOINT1 and TIMEPOINT2. The unit for the interval is given by the first argument, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR. See also the Time Interval and Point Unit Specifiers table.

    +

    E.g., timestampDiff(DAY, '2003-01-02 10:00:00'.toTimestamp, '2003-01-03 10:00:00'.toTimestamp) leads to 1.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Temporal functionsDescription
    + {% highlight scala %} +STRING.toDate +{% endhighlight %} + +

    Returns a SQL date parsed from STRING in form of "yyyy-MM-dd".

    +
    + {% highlight scala %} +STRING.toTime +{% endhighlight %} + +

    Returns a SQL time parsed from STRING in form of "HH:mm:ss".

    +
    + {% highlight scala %} +STRING.toTimestamp +{% endhighlight %} + +

    Returns a SQL timestamp parsed from STRING in form of "yyyy-MM-dd HH:mm:ss[.SSS]".

    +
    + {% highlight scala %} +NUMERIC.year +NUMERIC.years +{% endhighlight %} + +

    Creates an interval of months for NUMERIC years.

    +
    + {% highlight scala %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + +

    Creates an interval of months for NUMERIC quarters.

    +

    E.g., 2.quarters returns 6.

    +
    + {% highlight scala %} +NUMERIC.month +NUMERIC.months +{% endhighlight %} + +

    Creates an interval of NUMERIC months.

    +
    + {% highlight scala %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC weeks.

    +

    E.g., 2.weeks returns 1209600000.

    +
    + {% highlight scala %} +NUMERIC.day +NUMERIC.days +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC days.

    +
    + {% highlight scala %} +NUMERIC.hour +NUMERIC.hours +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC hours.

    +
    + {% highlight scala %} +NUMERIC.minute +NUMERIC.minutes +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC minutes.

    +
    + {% highlight scala %} +NUMERIC.second +NUMERIC.seconds +{% endhighlight %} + +

    Creates an interval of milliseconds for NUMERIC seconds.

    +
    + {% highlight scala %} +NUMERIC.milli +NUMERIC.millis +{% endhighlight %} + +

    Creates an interval of NUMERIC milliseconds.

    +
    + {% highlight scala %} +currentDate() +{% endhighlight %} + +

    Returns the current SQL date in the UTC time zone.

    +
    + {% highlight scala %} +currentTime() +{% endhighlight %} + +

    Returns the current SQL time in the UTC time zone.

    +
    + {% highlight scala %} +currentTimestamp() +{% endhighlight %} + +

    Returns the current SQL timestamp in the UTC time zone.

    +
    + {% highlight scala %} +localTime() +{% endhighlight %} + +

    Returns the current SQL time in local time zone.

    +
    + {% highlight scala %} +localTimestamp() +{% endhighlight %} + +

    Returns the current SQL timestamp in local time zone.

    +
    + {% highlight scala %} +TEMPORAL.extract(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a long value extracted from the TIMEINTERVALUNIT part of temporal.

    +

    E.g., "2006-06-05".toDate.extract(TimeIntervalUnit.DAY) returns 5; "2006-06-05".toDate.extract(QUARTER) returns 2.

    +
    + {% highlight scala %} +TIMEPOINT.floor(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a value that rounds TIMEPOINT down to the time unit TIMEINTERVALUNIT.

    +

    E.g., "12:44:31".toDate.floor(TimeIntervalUnit.MINUTE) returns 12:44:00.

    +
    + {% highlight scala %} +TIMEPOINT.ceil(TIMEINTERVALUNIT) +{% endhighlight %} + +

    Returns a value that rounds TIMEPOINT up to the time unit TIMEINTERVALUNIT.

    +

    E.g., "12:44:31".toTime.floor(TimeIntervalUnit.MINUTE) returns 12:45:00.

    +
    + {% highlight scala %} +temporalOverlaps(TIMEPOINT1, TEMPORAL1, TIMEPOINT2, TEMPORAL2) +{% endhighlight %} + +

    Returns TRUE if two time intervals defined by (TIMEPOINT1, TEMPORAL1) and (TIMEPOINT2, TEMPORAL2) overlap. The temporal values could be either a time point or a time interval.

    +

    E.g., temporalOverlaps("2:55:00".toTime, 1.hour, "3:30:00".toTime, 2.hour) returns TRUE.

    +
    + {% highlight scala %} +dateFormat(TIMESTAMP, STRING) +{% endhighlight %} + +

    Attention This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.

    +
    + {% highlight scala %} +timestampDiff(TIMEPOINTUNIT, TIMEPOINT1, TIMEPOINT2) +{% endhighlight %} + +

    Returns the (signed) number of TIMEPOINTUNIT between TIMEPOINT1 and TIMEPOINT2. The unit for the interval is given by the first argument, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR. See also the Time Interval and Point Unit Specifiers table.

    +

    E.g., timestampDiff(TimePointUnit.DAY, '2003-01-02 10:00:00'.toTimestamp, '2003-01-03 10:00:00'.toTimestamp) leads to 1.

    +
    +
    +
    + +{% top %} + +### Conditional Functions + +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Conditional functionsDescription
    + {% highlight text %} +CASE value +WHEN value1_1 [, value1_2 ]* THEN result1 +[ WHEN value2_1 [, value2_2 ]* THEN result2 ]* +[ ELSE resultZ ] +END +{% endhighlight %} + +

    Returns resultX when the first time value is contained in (valueX_1, valueX_2, ...). + When no value matches, returns resultZ if it is provided and returns NULL otherwise.

    +
    + {% highlight text %} +CASE +WHEN condition1 THEN result1 +[ WHEN condition2 THEN result2 ]* +[ ELSE resultZ ] +END +{% endhighlight %} + +

    Returns resultX when the first conditionX is met. + When no condition is met, returns resultZ if it is provided and returns NULL otherwise.

    +
    + {% highlight text %} +NULLIF(value1, value2) +{% endhighlight %} + +

    Returns NULL if value1 is equal to value2; returns value1 otherwise.

    +

    E.g., NULLIF(5, 5) returns NULL; NULLIF(5, 0) returns 5.

    +
    + {% highlight text %} +COALESCE(value1, value2 [, value3 ]* ) +{% endhighlight %} + +

    Returns the first value that is not NULL from value1, value2, ....

    +

    E.g., COALESCE(NULL, 5) returns 5.

    +
    +
    + +
    + + + + + + + + + + + + + + +
    Conditional functionsDescription
    + {% highlight java %} +BOOLEAN.?(VALUE1, VALUE2) +{% endhighlight %} + +

    Returns VALUE1 if BOOLEAN evaluates to TRUE; returns VALUE2 otherwise.

    +

    E.g., (42 > 5).?('A', 'B') returns "A".

    +
    +
    + +
    + + + + + + + + + + + + + + +
    Conditional functionsDescription
    + {% highlight scala %} +BOOLEAN.?(VALUE1, VALUE2) +{% endhighlight %} + +

    Returns VALUE1 if BOOLEAN evaluates to TRUE; returns VALUE2 otherwise.

    +

    E.g., (42 > 5).?("A", "B") returns "A".

    +
    +
    +
    + +{% top %} + +### Type Conversion Functions + +
    + +
    + + + + + + + + + + + + + + +
    Type conversion functionsDescription
    + {% highlight text %} +CAST(value AS type) +{% endhighlight %} + +

    Returns a new value being cast to type type. See the supported types here.

    +

    E.g., CAST('42' AS INT) returns 42; CAST(NULL AS VARCHAR) returns NULL of type VARCHAR.

    +
    +
    + +
    + + + + + + + + + + + + + + +
    Type conversion functionsDescription
    + {% highlight java %} +ANY.cast(TYPE) +{% endhighlight %} + +

    Returns a new ANY being cast to type TYPE. See the supported types here.

    +

    E.g., '42'.cast(INT) returns 42; Null(STRING) returns NULL of type STRING.

    +
    +
    + +
    + + + + + + + + + + + + + + +
    Type conversion functionsDescription
    + {% highlight scala %} +ANY.cast(TYPE) +{% endhighlight %} + +

    Returns a new ANY being cast to type TYPE. See the supported types here.

    +

    E.g., "42".cast(Types.INT) returns 42; Null(Types.STRING) returns NULL of type STRING.

    +
    + +
    +
    + +{% top %} + +### Collection Functions + +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Collection functionsDescription
    + {% highlight text %} +CARDINALITY(array) +{% endhighlight %} + +

    Returns the number of elements in array.

    +
    + {% highlight text %} +array ‘[’ integer ‘]’ +{% endhighlight %} + +

    Returns the element at position integer in array. The index starts from 1.

    +
    + {% highlight text %} +ELEMENT(array) +{% endhighlight %} + +

    Returns the sole element of array (whose cardinality should be one); returns NULL if array is empty. Throws an exception if array has more than one element.

    +
    + {% highlight text %} +CARDINALITY(map) +{% endhighlight %} + +

    Returns the number of entries in map.

    +
    + {% highlight text %} +map ‘[’ value ‘]’ +{% endhighlight %} + +

    Returns the value specified by key value in map.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Collection functionsDescription
    + {% highlight java %} +ARRAY.cardinality() +{% endhighlight %} + +

    Returns the number of elements in ARRAY.

    +
    + {% highlight java %} +ARRAY.at(INT) +{% endhighlight %} + +

    Returns the element at position INT in ARRAY. The index starts from 1.

    +
    + {% highlight java %} +ARRAY.element() +{% endhighlight %} + +

    Returns the sole element of ARRAY (whose cardinality should be one); returns NULL if ARRAY is empty. Throws an exception if ARRAY has more than one element.

    +
    + {% highlight java %} +MAP.cardinality() +{% endhighlight %} + +

    Returns the number of entries in MAP.

    +
    + {% highlight java %} +MAP.at(ANY) +{% endhighlight %} + +

    Returns the value specified by key ANY in MAP.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Collection functionsDescription
    + {% highlight scala %} +ARRAY.cardinality() +{% endhighlight %} + +

    Returns the number of elements in ARRAY.

    +
    + {% highlight scala %} +ARRAY.at(INT) +{% endhighlight %} + +

    Returns the element at position INT in ARRAY. The index starts from 1.

    +
    + {% highlight scala %} +ARRAY.element() +{% endhighlight %} + +

    Returns the sole element of ARRAY (whose cardinality should be one); returns NULL if ARRAY is empty. Throws an exception if ARRAY has more than one element.

    +
    + {% highlight scala %} +MAP.cardinality() +{% endhighlight %} + +

    Returns the number of entries in MAP.

    +
    + {% highlight scala %} +MAP.at(ANY) +{% endhighlight %} + +

    Returns the value specified by key ANY in MAP.

    +
    +
    +
    + + +### Value Construction Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    Value construction functionsDescription
    + {% highlight text %} +ROW(value1, [, value2]*) +(value1, [, value2]*) +{% endhighlight %} + +

    Returns a row created from a list of values (value1, value2,...).

    +
    + {% highlight text %} +ARRAY ‘[’ value1 [, value2 ]* ‘]’ +{% endhighlight %} + +

    Returns an array created from a list of values (value1, value2, ...).

    +
    + {% highlight text %} +MAP ‘[’ value1, value2 [, value3, value4 ]* ‘]’ +{% endhighlight %} + +

    Returns a map created from a list of key-value pairs ((value1, value2), (value3, value4), ...).

    +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Value constructor functionsDescription
    + {% highlight java %} +row(ANY1, ANY2, ...) +{% endhighlight %} + +

    Returns a row created from a list of object values (ANY1, ANY2, ...). Row is composite type that can be access via value access functions.

    +
    + {% highlight java %} +array(ANY1, ANY2, ...) +{% endhighlight %} + +

    Returns an array created from a list of object values (ANY1, ANY2, ...).

    +
    + {% highlight java %} +map(ANY1, ANY2, ANY3, ANY4, ...) +{% endhighlight %} + +

    Returns a map created from a list of key-value pairs ((ANY1, ANY2), (ANY3, ANY4), ...).

    +
    + {% highlight java %} +NUMERIC.rows +{% endhighlight %} + +

    Creates a NUMERIC interval of rows (commonly used in window creation).

    +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Value constructor functionsDescription
    + {% highlight scala %} +row(ANY1, ANY2, ...) +{% endhighlight %} + +

    Returns a row created from a list of object values (ANY1, ANY2, ...). Row is composite type that can be access via value access functions.

    +
    + {% highlight scala %} +array(ANY1, ANY2, ...) +{% endhighlight %} + +

    Returns an array created from a list of object values (ANY1, ANY2, ...).

    +
    + {% highlight scala %} +map(ANY1, ANY2, ANY3, ANY4, ...) +{% endhighlight %} + +

    Returns a map created from a list of key-value pairs ((ANY1, ANY2), (ANY3, ANY4), ...).

    +
    + {% highlight scala %} +NUMERIC.rows +{% endhighlight %} + +

    Creates a NUMERIC interval of rows (commonly used in window creation).

    +
    + +
    +
    + +{% top %} + +### Value Access Functions + +
    +
    + + + + + + + + + + + + + + + + + + + +
    Value access functionsDescription
    + {% highlight text %} +tableName.compositeType.field +{% endhighlight %} + +

    Returns the value of a field from a Flink composite type (e.g., Tuple, POJO) by name.

    +
    + {% highlight text %} +tableName.compositeType.* +{% endhighlight %} + +

    Returns a flat representation of a Flink composite type (e.g., Tuple, POJO) that converts each of its direct subtype into a separate field. + In most cases the fields of the flat representation are named similarly to the original fields but with a dollar separator (e.g., mypojo$mytuple$f0).

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + +
    Value access functionsDescription
    + {% highlight java %} +COMPOSITE.get(STRING) +COMPOSITE.get(INT) +{% endhighlight %} + +

    Returns the value of a field from a Flink composite type (e.g., Tuple, POJO) by name or index.

    +

    E.g., pojo.get('myField') or tuple.get(0).

    +
    + {% highlight java %} +ANY.flatten() +{% endhighlight %} + +

    Returns a flat representation of a Flink composite type (e.g., Tuple, POJO) that converts each of its direct subtype into a separate field. + In most cases the fields of the flat representation are named similarly to the original fields but with a dollar separator (e.g., mypojo$mytuple$f0).

    +
    +
    +
    + + + + + + + + + + + + + + + + + + + +
    Value access functionsDescription
    + {% highlight scala %} +COMPOSITE.get(STRING) +COMPOSITE.get(INT) +{% endhighlight %} + +

    Returns the value of a field from a Flink composite type (e.g., Tuple, POJO) by name or index.

    +

    E.g., 'pojo.get("myField") or 'tuple.get(0).

    +
    + {% highlight scala %} +ANY.flatten() +{% endhighlight %} + +

    Returns a flat representation of a Flink composite type (e.g., Tuple, POJO) that converts each of its direct subtype into a separate field. + In most cases the fields of the flat representation are named similarly to the original fields but with a dollar separator (e.g., mypojo$mytuple$f0).

    +
    + +
    +
    + +{% top %} + +### Grouping Functions + +
    +
    + + + + + + + + + + + + + + + + + + + +
    Grouping functionsDescription
    + {% highlight text %} +GROUP_ID() +{% endhighlight %} + +

    Returns an integer that uniquely identifies the combination of grouping keys.

    +
    + {% highlight text %} +GROUPING(expression1 [, expression2]* ) +GROUPING_ID(expression1 [, expression2]* ) +{% endhighlight %} + +

    Returns a bit vector of the given grouping expressions.

    +
    +
    + +
    + + + + + + + + + +
    Grouping functionsDescription
    +
    + +
    + + + + + + + + + +
    Grouping functionsDescription
    +
    +
    + +### Hash Functions + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Hash functionsDescription
    + {% highlight text %} +MD5(string) +{% endhighlight %} + +

    Returns the MD5 hash of string as a string of 32 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA1(string) +{% endhighlight %} + +

    Returns the SHA-1 hash of string as a string of 40 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA224(string) +{% endhighlight %} + +

    Returns the SHA-224 hash of string as a string of 56 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA256(string) +{% endhighlight %} + +

    Returns the SHA-256 hash of string as a string of 64 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA384(string) +{% endhighlight %} + +

    Returns the SHA-384 hash of string as a string of 96 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA512(string) +{% endhighlight %} + +

    Returns the SHA-512 hash of string as a string of 128 hexadecimal digits; returns NULL if string is NULL.

    +
    + {% highlight text %} +SHA2(string, hashLength) +{% endhighlight %} + +

    Returns the hash using the SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, or SHA-512). The first argument string is the string to be hashed and the second argument hashLength is the bit length of the result (224, 256, 384, or 512). Returns NULL if string or hashLength is NULL. +

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Hash functionsDescription
    + {% highlight java %} +STRING.md5() +{% endhighlight %} + +

    Returns the MD5 hash of STRING as a string of 32 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha1() +{% endhighlight %} + +

    Returns the SHA-1 hash of STRING as a string of 40 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha224() +{% endhighlight %} + +

    Returns the SHA-224 hash of STRING as a string of 56 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha256() +{% endhighlight %} + +

    Returns the SHA-256 hash of STRING as a string of 64 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha384() +{% endhighlight %} + +

    Returns the SHA-384 hash of STRING as a string of 96 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha512() +{% endhighlight %} + +

    Returns the SHA-512 hash of STRING as a string of 128 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight java %} +STRING.sha2(INT) +{% endhighlight %} + +

    Returns the SHA-2 family (SHA-224, SHA-256, SHA-384, or SHA-512) hashed value specified by INT (which could be 224, 256, 384, or 512) for STRING. Returns NULL if STRING or INT is NULL. +

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Hash functionsDescription
    + {% highlight scala %} +STRING.md5() +{% endhighlight %} + +

    Returns the MD5 hash of STRING as a string of 32 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha1() +{% endhighlight %} + +

    Returns the SHA-1 hash of STRING as a string of 40 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha224() +{% endhighlight %} + +

    Returns the SHA-224 hash of STRING as a string of 56 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha256() +{% endhighlight %} + +

    Returns the SHA-256 hash of STRING as a string of 64 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha384() +{% endhighlight %} + +

    Returns the SHA-384 hash of STRING as a string of 96 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha512() +{% endhighlight %} + +

    Returns the SHA-512 hash of STRING as a string of 128 hexadecimal digits; returns NULL if STRING is NULL.

    +
    + {% highlight scala %} +STRING.sha2(INT) +{% endhighlight %} + +

    Returns the SHA-2 family (SHA-224, SHA-256, SHA-384, or SHA-512) hashed value specified by INT (which could be 224, 256, 384, or 512) for STRING. Returns NULL if STRING or INT is NULL. +

    +
    +
    +
    + +{% top %} + +### Auxiliary Functions + +
    + +
    + + + + + + + + + + +
    Auxiliary functionsDescription
    +
    + +
    + + + + + + + + + + + + + + +
    Auxiliary functionsDescription
    + {% highlight java %} +ANY.as(NAME1, NAME2, ...) +{% endhighlight %} + +

    Specifies a name for ANY (a field). Additional names can be specified if the expression expands to multiple fields.

    +
    + +
    +
    + + + + + + + + + + + + + + + +
    Auxiliary functionsDescription
    + {% highlight scala %} +ANY.as(NAME1, NAME2, ...) +{% endhighlight %} + +

    Specifies a name for ANY (a field). Additional names can be specified if the expression expands to multiple fields.

    +
    +
    + +
    + +Aggregate Functions +------------------- + +The aggregate functions take an expression across all the rows as the input and return a single aggregated value as the result. + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Aggregate functionsDescription
    + {% highlight text %} +COUNT([ ALL ] expression | DISTINCT expression1 [, expression2]*) +{% endhighlight %} + +

    By default or with ALL, returns the number of input rows for which expression is not NULL. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +COUNT(*) +COUNT(1) +{% endhighlight %} + +

    Returns the number of input rows.

    +
    + {% highlight text %} +AVG([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the average (arithmetic mean) of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +SUM([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the sum of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +MAX([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the maximum value of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +MIN([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the minimum value of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +STDDEV_POP([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the population standard deviation of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +STDDEV_SAMP([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the sample standard deviation of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +VAR_POP([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the population variance (square of the population standard deviation) of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +VAR_SAMP([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns the sample variance (square of the sample standard deviation) of expression across all input rows. Use DISTINCT for one unique instance of each value.

    +
    + {% highlight text %} +COLLECT([ ALL | DISTINCT ] expression) +{% endhighlight %} + +

    By default or with keyword ALL, returns a multiset of expression across all input rows. NULL values will be ignored. Use DISTINCT for one unique instance of each value.

    +
    + +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Aggregate functionsDescription
    + {% highlight java %} +FIELD.count +{% endhighlight %} + +

    Returns the number of input rows for which FIELD is not NULL.

    +
    + {% highlight java %} +FIELD.avg +{% endhighlight %} + +

    Returns the average (arithmetic mean) of FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.sum +{% endhighlight %} + +

    Returns the sum of numeric field FIELD across all input rows. If all values are NULL, returns NULL.

    +
    + {% highlight java %} +FIELD.sum0 +{% endhighlight %} + +

    Returns the sum of numeric field FIELD across all input rows. If all values are NULL, returns 0.

    +
    + {% highlight java %} +FIELD.max +{% endhighlight %} + +

    Returns the maximum value of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.min +{% endhighlight %} + +

    Returns the minimum value of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.stddevPop +{% endhighlight %} + +

    Returns the population standard deviation of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.stddevSamp +{% endhighlight %} + +

    Returns the sample standard deviation of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.varPop +{% endhighlight %} + +

    Returns the population variance (square of the population standard deviation) of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.varSamp +{% endhighlight %} + +

    Returns the sample variance (square of the sample standard deviation) of numeric field FIELD across all input rows.

    +
    + {% highlight java %} +FIELD.collect +{% endhighlight %} + +

    Returns a multiset of FIELD across all input rows.

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Aggregate functionsDescription
    + {% highlight scala %} +FIELD.count +{% endhighlight %} + +

    Returns the number of input rows for which FIELD is not NULL.

    +
    + {% highlight scala %} +FIELD.avg +{% endhighlight %} + +

    Returns the average (arithmetic mean) of FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.sum +{% endhighlight %} + +

    Returns the sum of numeric field FIELD across all input rows. If all values are NULL, returns NULL.

    +
    + {% highlight scala %} +FIELD.sum0 +{% endhighlight %} + +

    Returns the sum of numeric field FIELD across all input rows. If all values are NULL, returns 0.

    +
    + {% highlight scala %} +FIELD.max +{% endhighlight %} + +

    Returns the maximum value of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.min +{% endhighlight %} + +

    Returns the minimum value of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.stddevPop +{% endhighlight %} + +

    Returns the population standard deviation of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.stddevSamp +{% endhighlight %} + +

    Returns the sample standard deviation of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.varPop +{% endhighlight %} + +

    Returns the population variance (square of the population standard deviation) of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.varSamp +{% endhighlight %} + +

    Returns the sample variance (square of the sample standard deviation) of numeric field FIELD across all input rows.

    +
    + {% highlight scala %} +FIELD.collect +{% endhighlight %} + +

    Returns a multiset of FIELD across all input rows.

    +
    +
    +
    + +{% top %} + +Date Format Specifiers +---------------------- + +The following table lists specifiers for date format functions. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    SpecifierDescription
    {% highlight text %}%a{% endhighlight %}Abbreviated weekday name (Sun .. Sat)
    {% highlight text %}%b{% endhighlight %}Abbreviated month name (Jan .. Dec)
    {% highlight text %}%c{% endhighlight %}Month, numeric (1 .. 12)
    {% highlight text %}%D{% endhighlight %}Day of the month with English suffix (0th, 1st, 2nd, 3rd, ...)
    {% highlight text %}%d{% endhighlight %}Day of the month, numeric (01 .. 31)
    {% highlight text %}%e{% endhighlight %}Day of the month, numeric (1 .. 31)
    {% highlight text %}%f{% endhighlight %}Fraction of second (6 digits for printing: 000000 .. 999000; 1 - 9 digits for parsing: 0 .. 999999999) (Timestamp is truncated to milliseconds.)
    {% highlight text %}%H{% endhighlight %}Hour (00 .. 23)
    {% highlight text %}%h{% endhighlight %}Hour (01 .. 12)
    {% highlight text %}%I{% endhighlight %}Hour (01 .. 12)
    {% highlight text %}%i{% endhighlight %}Minutes, numeric (00 .. 59)
    {% highlight text %}%j{% endhighlight %}Day of year (001 .. 366)
    {% highlight text %}%k{% endhighlight %}Hour (0 .. 23)
    {% highlight text %}%l{% endhighlight %}Hour (1 .. 12)
    {% highlight text %}%M{% endhighlight %}Month name (January .. December)
    {% highlight text %}%m{% endhighlight %}Month, numeric (01 .. 12)
    {% highlight text %}%p{% endhighlight %}AM or PM
    {% highlight text %}%r{% endhighlight %}Time, 12-hour (hh:mm:ss followed by AM or PM)
    {% highlight text %}%S{% endhighlight %}Seconds (00 .. 59)
    {% highlight text %}%s{% endhighlight %}Seconds (00 .. 59)
    {% highlight text %}%T{% endhighlight %}Time, 24-hour (hh:mm:ss)
    {% highlight text %}%U{% endhighlight %}Week (00 .. 53), where Sunday is the first day of the week
    {% highlight text %}%u{% endhighlight %}Week (00 .. 53), where Monday is the first day of the week
    {% highlight text %}%V{% endhighlight %}Week (01 .. 53), where Sunday is the first day of the week; used with %X
    {% highlight text %}%v{% endhighlight %}Week (01 .. 53), where Monday is the first day of the week; used with %x
    {% highlight text %}%W{% endhighlight %}Weekday name (Sunday .. Saturday)
    {% highlight text %}%w{% endhighlight %}Day of the week (0 .. 6), where Sunday is the first day of the week
    {% highlight text %}%X{% endhighlight %}Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V
    {% highlight text %}%x{% endhighlight %}Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v
    {% highlight text %}%Y{% endhighlight %}Year, numeric, four digits
    {% highlight text %}%y{% endhighlight %}Year, numeric (two digits)
    {% highlight text %}%%{% endhighlight %}A literal % character
    {% highlight text %}%x{% endhighlight %}x, for any x not listed above
    + +{% top %} + +Time Interval and Point Unit Specifiers +--------------------------------------- + +The following table lists specifiers for time interval and time point units. + +For Table API, please use `_` for spaces (e.g., `DAY_TO_HOUR`). + +| Time Interval Unit | Time Point Unit | +| :----------------------- | :----------------------------- | +| `MILLENIUM` _(SQL-only)_ | | +| `CENTURY` _(SQL-only)_ | | +| `YEAR` | `YEAR` | +| `YEAR TO MONTH` | | +| `QUARTER` | `QUARTER` | +| `MONTH` | `MONTH` | +| `WEEK` | `WEEK` | +| `DAY` | `DAY` | +| `DAY TO HOUR` | | +| `DAY TO MINUTE` | | +| `DAY TO SECOND` | | +| `HOUR` | `HOUR` | +| `HOUR TO MINUTE` | | +| `HOUR TO SECOND` | | +| `MINUTE` | `MINUTE` | +| `MINUTE TO SECOND` | | +| `SECOND` | `SECOND` | +| | `MILLISECOND` | +| | `MICROSECOND` | +| `DOY` _(SQL-only)_ | | +| `DOW` _(SQL-only)_ | | +| | `SQL_TSI_YEAR` _(SQL-only)_ | +| | `SQL_TSI_QUARTER` _(SQL-only)_ | +| | `SQL_TSI_MONTH` _(SQL-only)_ | +| | `SQL_TSI_WEEK` _(SQL-only)_ | +| | `SQL_TSI_DAY` _(SQL-only)_ | +| | `SQL_TSI_HOUR` _(SQL-only)_ | +| | `SQL_TSI_MINUTE` _(SQL-only)_ | +| | `SQL_TSI_SECOND ` _(SQL-only)_ | + +{% top %} diff --git a/docs/dev/table/index.md b/docs/dev/table/index.md index 5845c95099cbb3..0b80801badbef9 100644 --- a/docs/dev/table/index.md +++ b/docs/dev/table/index.md @@ -2,7 +2,7 @@ title: "Table API & SQL" nav-id: tableapi nav-parent_id: dev -is_beta: true +is_beta: false nav-show_overview: true nav-pos: 35 --- @@ -73,10 +73,11 @@ Where to go next? ----------------- * [Concepts & Common API]({{ site.baseurl }}/dev/table/common.html): Shared concepts and APIs of the Table API and SQL. -* [Streaming Table API & SQL]({{ site.baseurl }}/dev/table/streaming.html): Streaming-specific documentation for the Table API or SQL such as configuration of time attributes and handling of updating results. +* [Streaming Concepts]({{ site.baseurl }}/dev/table/streaming): Streaming-specific documentation for the Table API or SQL such as configuration of time attributes and handling of updating results. +* [Connect to External Systems]({{ site.baseurl }}/dev/table/functions.html): Available connectors and formats for reading and writing data to external systems. * [Table API]({{ site.baseurl }}/dev/table/tableApi.html): Supported operations and API for the Table API. -* [SQL]({{ site.baseurl }}/dev/table/sql.html): Supported operations and syntax for SQL -* [Table Sources & Sinks]({{ site.baseurl }}/dev/table/sourceSinks.html): Reading tables from and emitting tables to external storage systems. -* [User-Defined Functions]({{ site.baseurl }}/dev/table/udfs.html): Definition and usage of user-defined functions. +* [SQL]({{ site.baseurl }}/dev/table/sql.html): Supported operations and syntax for SQL. +* [Built-in Functions]({{ site.baseurl }}/dev/table/functions.html): Supported functions in Table API and SQL. +* [SQL Client]({{ site.baseurl }}/dev/table/sqlClient.html): Play around with Flink SQL and submit a table program to a cluster without programming knowledge. -{% top %} \ No newline at end of file +{% top %} diff --git a/docs/dev/table/sourceSinks.md b/docs/dev/table/sourceSinks.md index 792f19d260a705..483ef20e64ef4d 100644 --- a/docs/dev/table/sourceSinks.md +++ b/docs/dev/table/sourceSinks.md @@ -1,5 +1,5 @@ --- -title: "Table Sources & Sinks" +title: "User-defined Sources & Sinks" nav-parent_id: tableapi nav-pos: 40 --- @@ -22,751 +22,17 @@ specific language governing permissions and limitations under the License. --> -A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can accessed by [Table API](tableApi.html) or [SQL](sql.html) queries. +A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can be accessed by [Table API](tableApi.html) or [SQL](sql.html) queries. -A TableSink [emits a Table](common.html#emit-a-table) to an external storage system, such as a database, key-value store, message queue, or file system (in different encodings, e.g., CSV, Parquet, or ORC). +A `TableSink` [emits a Table](common.html#emit-a-table) to an external storage system, such as a database, key-value store, message queue, or file system (in different encodings, e.g., CSV, Parquet, or ORC). -Have a look at the [common concepts and API](common.html) page for details how to [register a TableSource](common.html#register-a-tablesource) and how to [emit a Table through a TableSink](common.html#emit-a-table). +A `TableFactory` allows for separating the declaration of a connection to an external system from the actual implementation. A table factory creates configured instances of table sources and sinks from normalized, string-based properties. The properties can be generated programmatically using a `Descriptor` or via YAML configuration files for the [SQL Client](sqlClient.html). + +Have a look at the [common concepts and API](common.html) page for details how to [register a TableSource](common.html#register-a-tablesource) and how to [emit a Table through a TableSink](common.html#emit-a-table). See the [built-in sources, sinks, and formats](connect.html) page for examples how to use factories. * This will be replaced by the TOC {:toc} -Provided TableSources ---------------------- - -Currently, Flink provides the `CsvTableSource` to read CSV files and a few table sources to read JSON or Avro data from Kafka. -A custom `TableSource` can be defined by implementing the `BatchTableSource` or `StreamTableSource` interface. See section on [defining a custom TableSource](#define-a-tablesource) for details. - -| **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description** -| `Kafka011AvroTableSource` | `flink-connector-kafka-0.11` | N | Y | A `TableSource` for Avro-encoded Kafka 0.11 topics. -| `Kafka011JsonTableSource` | `flink-connector-kafka-0.11` | N | Y | A `TableSource` for flat Json-encoded Kafka 0.11 topics. -| `Kafka010AvroTableSource` | `flink-connector-kafka-0.10` | N | Y | A `TableSource` for Avro-encoded Kafka 0.10 topics. -| `Kafka010JsonTableSource` | `flink-connector-kafka-0.10` | N | Y | A `TableSource` for flat Json-encoded Kafka 0.10 topics. -| `Kafka09AvroTableSource` | `flink-connector-kafka-0.9` | N | Y | A `TableSource` for Avro-encoded Kafka 0.9 topics. -| `Kafka09JsonTableSource` | `flink-connector-kafka-0.9` | N | Y | A `TableSource` for flat Json-encoded Kafka 0.9 topics. -| `Kafka08AvroTableSource` | `flink-connector-kafka-0.8` | N | Y | A `TableSource` for Avro-encoded Kafka 0.8 topics. -| `Kafka08JsonTableSource` | `flink-connector-kafka-0.8` | N | Y | A `TableSource` for flat Json-encoded Kafka 0.8 topics. -| `CsvTableSource` | `flink-table` | Y | Y | A simple `TableSource` for CSV files. -| `OrcTableSource` | `flink-orc` | Y | N | A `TableSource` for ORC files. - -All sources that come with the `flink-table` dependency are directly available for Table API or SQL programs. For all other table sources, you have to add the respective dependency in addition to the `flink-table` dependency. - -{% top %} - -### KafkaJsonTableSource - -A `KafkaJsonTableSource` ingests JSON-encoded messages from a Kafka topic. Currently, only JSON records with flat (non-nested) schema are supported. - -A `KafkaJsonTableSource` is created and configured using a builder. The following example shows how to create a `KafkaJsonTableSource` with basic properties: - -
    -
    -{% highlight java %} -// create builder -KafkaTableSource source = Kafka010JsonTableSource.builder() - // set Kafka topic - .forTopic("sensors") - // set Kafka consumer properties - .withKafkaProperties(kafkaProps) - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temp", Types.DOUBLE()) - .field("time", Types.SQL_TIMESTAMP()).build()) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -// create builder -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // set Kafka topic - .forTopic("sensors") - // set Kafka consumer properties - .withKafkaProperties(kafkaProps) - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temp", Types.DOUBLE) - .field("time", Types.SQL_TIMESTAMP).build()) - .build() -{% endhighlight %} -
    -
    - -#### Optional Configuration - -* **Time Attributes:** Please see the sections on [configuring a rowtime attribute](#configure-a-rowtime-attribute) and [configuring a processing time attribute](#configure-a-processing-time-attribute). - -* **Explicit JSON parse schema:** By default, the JSON records are parsed with the table schema. You can configure an explicit JSON schema and provide a mapping from table schema fields to JSON fields as shown in the following example. - -
    -
    -{% highlight java %} -Map mapping = new HashMap<>(); -mapping.put("sensorId", "id"); -mapping.put("temperature", "temp"); - -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temperature", Types.DOUBLE()).build()) - // set JSON parsing schema - .forJsonSchema(TableSchema.builder() - .field("id", Types.LONG()) - .field("temp", Types.DOUBLE()).build()) - // set mapping from table fields to JSON fields - .withTableToJsonMapping(mapping) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temperature", Types.DOUBLE).build()) - // set JSON parsing schema - .forJsonSchema(TableSchema.builder() - .field("id", Types.LONG) - .field("temp", Types.DOUBLE).build()) - // set mapping from table fields to JSON fields - .withTableToJsonMapping(Map( - "sensorId" -> "id", - "temperature" -> "temp").asJava) - .build() -{% endhighlight %} -
    -
    - -* **Missing Field Handling:** By default, a missing JSON field is set to `null`. You can enable strict JSON parsing that will cancel the source (and query) if a field is missing. - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - // configure missing field behavior - .failOnMissingField(true) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - // configure missing field behavior - .failOnMissingField(true) - .build() -{% endhighlight %} -
    -
    - -* **Specify the start reading position:** By default, the table source will start reading data from the committed group offsets in Zookeeper or Kafka brokers. You can specify other start positions via the builder's methods, which correspond to the configurations in section [Kafka Consumers Start Position Configuration](../connectors/kafka.html#kafka-consumers-start-position-configuration). - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - // start reading from the earliest offset - .fromEarliest() - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - // start reading from the earliest offset - .fromEarliest() - .build() -{% endhighlight %} -
    -
    - -{% top %} - -### KafkaAvroTableSource - -A `KafkaAvroTableSource` ingests Avro-encoded records from a Kafka topic. - -A `KafkaAvroTableSource` is created and configured using a builder. The following example shows how to create a `KafkaAvroTableSource` with basic properties: - -
    -
    -{% highlight java %} -// create builder -KafkaTableSource source = Kafka010AvroTableSource.builder() - // set Kafka topic - .forTopic("sensors") - // set Kafka consumer properties - .withKafkaProperties(kafkaProps) - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temp", Types.DOUBLE()) - .field("time", Types.SQL_TIMESTAMP()).build()) - // set class of Avro record - .forAvroRecordClass(SensorReading.class) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -// create builder -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // set Kafka topic - .forTopic("sensors") - // set Kafka consumer properties - .withKafkaProperties(kafkaProps) - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temp", Types.DOUBLE) - .field("time", Types.SQL_TIMESTAMP).build()) - // set class of Avro record - .forAvroRecordClass(classOf[SensorReading]) - .build() -{% endhighlight %} -
    -
    - -**NOTE:** The specified Avro record class must provide all fields of the table schema with corresponding type. - -#### Optional Configuration - -* **Time Attributes:** Please see the sections on [configuring a rowtime attribute](#configure-a-rowtime-attribute) and [configuring a processing time attribute](#configure-a-processing-time-attribute). - -* **Explicit Schema Field to Avro Mapping:** By default, all fields of the table schema are mapped by name to fields of the Avro records. If the fields in the Avro records have different names, a mapping from table schema fields to Avro fields can be specified. - -
    -
    -{% highlight java %} -Map mapping = new HashMap<>(); -mapping.put("sensorId", "id"); -mapping.put("temperature", "temp"); - -KafkaTableSource source = Kafka010AvroTableSource.builder() - // ... - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temperature", Types.DOUBLE()).build()) - // set class of Avro record with fields [id, temp] - .forAvroRecordClass(SensorReading.class) - // set mapping from table fields to Avro fields - .withTableToAvroMapping(mapping) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010AvroTableSource.builder() - // ... - // set Table schema - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temperature", Types.DOUBLE).build()) - // set class of Avro record with fields [id, temp] - .forAvroRecordClass(classOf[SensorReading]) - // set mapping from table fields to Avro fields - .withTableToAvroMapping(Map( - "sensorId" -> "id", - "temperature" -> "temp").asJava) - .build() -{% endhighlight %} -
    -
    - -* **Specify the start reading position:** By default, the table source will start reading data from the committed group offsets in Zookeeper or Kafka brokers. You can specify other start positions via the builder's methods, which correspond to the configurations in section [Kafka Consumers Start Position Configuration](../connectors/kafka.html#kafka-consumers-start-position-configuration). - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010AvroTableSource.builder() - // ... - // start reading from the earliest offset - .fromEarliest() - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010AvroTableSource.builder() - // ... - // start reading from the earliest offset - .fromEarliest() - .build() -{% endhighlight %} -
    -
    - -{% top %} - -### Configuring a Processing Time Attribute - -[Processing time attributes](streaming.html#processing-time) are commonly used in streaming queries. A processing time attribute returns the current wall-clock time of the operator that accesses it. - -Batch queries support processing time attributes as well. However, processing time attributes are initialized with the wall-clock time of the table scan operator and keep this value throughout the query evaluation. - -A table schema field of type `SQL_TIMESTAMP` can be declared as a processing time attribute as shown in the following example. - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temp", Types.DOUBLE()) - // field "ptime" is of type SQL_TIMESTAMP - .field("ptime", Types.SQL_TIMESTAMP()).build()) - // declare "ptime" as processing time attribute - .withProctimeAttribute("ptime") - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temp", Types.DOUBLE) - // field "ptime" is of type SQL_TIMESTAMP - .field("ptime", Types.SQL_TIMESTAMP).build()) - // declare "ptime" as processing time attribute - .withProctimeAttribute("ptime") - .build() -{% endhighlight %} -
    -
    - -{% top %} - -### Configuring a Rowtime Attribute - -[Rowtime attributes](streaming.html#event-time) are attributes of type `TIMESTAMP` and handled in a unified way in stream and batch queries. - -A table schema field of type `SQL_TIMESTAMP` can be declared as rowtime attribute by specifying - -* the name of the field, -* a `TimestampExtractor` that computes the actual value for the attribute (usually from one or more other attributes), and -* a `WatermarkStrategy` that specifies how watermarks are generated for the the rowtime attribute. - -The following example shows how to configure a rowtime attribute. - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temp", Types.DOUBLE()) - // field "rtime" is of type SQL_TIMESTAMP - .field("rtime", Types.SQL_TIMESTAMP()).build()) - .withRowtimeAttribute( - // "rtime" is rowtime attribute - "rtime", - // value of "rtime" is extracted from existing field with same name - new ExistingField("rtime"), - // values of "rtime" are at most out-of-order by 30 seconds - new BoundedOutOfOrderWatermarks(30000L)) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temp", Types.DOUBLE) - // field "rtime" is of type SQL_TIMESTAMP - .field("rtime", Types.SQL_TIMESTAMP).build()) - .withRowtimeAttribute( - // "rtime" is rowtime attribute - "rtime", - // value of "rtime" is extracted from existing field with same name - new ExistingField("rtime"), - // values of "rtime" are at most out-of-order by 30 seconds - new BoundedOutOfOrderTimestamps(30000L)) - .build() -{% endhighlight %} -
    -
    - -#### Extracting Kafka 0.10+ Timestamps into Rowtime Attribute - -Since Kafka 0.10, Kafka messages have a timestamp as metadata that specifies when the record was written into the Kafka topic. `KafkaTableSources` can assign Kafka's message timestamp as rowtime attribute as follows: - -
    -
    -{% highlight java %} -KafkaTableSource source = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG()) - .field("temp", Types.DOUBLE()) - // field "rtime" is of type SQL_TIMESTAMP - .field("rtime", Types.SQL_TIMESTAMP()).build()) - // use Kafka timestamp as rowtime attribute - .withKafkaTimestampAsRowtimeAttribute()( - // "rtime" is rowtime attribute - "rtime", - // values of "rtime" are ascending - new AscendingTimestamps()) - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val source: KafkaTableSource = Kafka010JsonTableSource.builder() - // ... - .withSchema(TableSchema.builder() - .field("sensorId", Types.LONG) - .field("temp", Types.DOUBLE) - // field "rtime" is of type SQL_TIMESTAMP - .field("rtime", Types.SQL_TIMESTAMP).build()) - // use Kafka timestamp as rowtime attribute - .withKafkaTimestampAsRowtimeAttribute()( - // "rtime" is rowtime attribute - "rtime", - // values of "rtime" are ascending - new AscendingTimestamps()) - .build() -{% endhighlight %} -
    -
    - -#### Provided TimestampExtractors - -Flink provides `TimestampExtractor` implementations for common use cases. -The following `TimestampExtractor` implementations are currently available: - -* `ExistingField(fieldName)`: Extracts the value of a rowtime attribute from an existing `LONG` or `SQL_TIMESTAMP` field. -* `StreamRecordTimestamp()`: Extracts the value of a rowtime attribute from the timestamp of the `DataStream` `StreamRecord`. Note, this `TimestampExtractor` is not available for batch table sources. - -A custom `TimestampExtractor` can be defined by implementing the corresponding interface. - -#### Provided WatermarkStrategies - -Flink provides `WatermarkStrategy` implementations for common use cases. -The following `WatermarkStrategy` implementations are currently available: - -* `AscendingTimestamps`: A watermark strategy for ascending timestamps. Records with timestamps that are out-of-order will be considered late. -* `BoundedOutOfOrderTimestamps(delay)`: A watermark strategy for timestamps that are at most out-of-order by the specified delay. - -A custom `WatermarkStrategy` can be defined by implementing the corresponding interface. - -{% top %} - -### CsvTableSource - -The `CsvTableSource` is already included in `flink-table` without additional dependencies. - -The easiest way to create a `CsvTableSource` is by using the enclosed builder `CsvTableSource.builder()`, the builder has the following methods to configure properties: - - - `path(String path)` Sets the path to the CSV file, required. - - `field(String fieldName, TypeInformation fieldType)` Adds a field with the field name and field type information, can be called multiple times, required. The call order of this method defines also the order of the fields in a row. - - `fieldDelimiter(String delim)` Sets the field delimiter, `","` by default. - - `lineDelimiter(String delim)` Sets the line delimiter, `"\n"` by default. - - `quoteCharacter(Character quote)` Sets the quote character for String values, `null` by default. - - `commentPrefix(String prefix)` Sets a prefix to indicate comments, `null` by default. - - `ignoreFirstLine()` Ignore the first line. Disabled by default. - - `ignoreParseErrors()` Skip records with parse error instead to fail. Throwing an exception by default. - -You can create the source as follows: - -
    -
    -{% highlight java %} -CsvTableSource csvTableSource = CsvTableSource - .builder() - .path("/path/to/your/file.csv") - .field("name", Types.STRING()) - .field("id", Types.INT()) - .field("score", Types.DOUBLE()) - .field("comments", Types.STRING()) - .fieldDelimiter("#") - .lineDelimiter("$") - .ignoreFirstLine() - .ignoreParseErrors() - .commentPrefix("%") - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val csvTableSource = CsvTableSource - .builder - .path("/path/to/your/file.csv") - .field("name", Types.STRING) - .field("id", Types.INT) - .field("score", Types.DOUBLE) - .field("comments", Types.STRING) - .fieldDelimiter("#") - .lineDelimiter("$") - .ignoreFirstLine - .ignoreParseErrors - .commentPrefix("%") - .build -{% endhighlight %} -
    -
    - -{% top %} - -### OrcTableSource - -The `OrcTableSource` reads [ORC files](https://orc.apache.org). ORC is a file format for structured data and stores the data in a compressed, columnar representation. ORC is very storage efficient and supports projection and filter push-down. - -An `OrcTableSource` is created as shown below: - -
    -
    -{% highlight java %} - -// create Hadoop Configuration -Configuration config = new Configuration(); - -OrcTableSource orcTableSource = OrcTableSource.builder() - // path to ORC file(s). NOTE: By default, directories are recursively scanned. - .path("file:///path/to/data") - // schema of ORC files - .forOrcSchema("struct>>") - // Hadoop configuration - .withConfiguration(config) - // build OrcTableSource - .build(); -{% endhighlight %} -
    - -
    -{% highlight scala %} - -// create Hadoop Configuration -val config = new Configuration() - -val orcTableSource = OrcTableSource.builder() - // path to ORC file(s). NOTE: By default, directories are recursively scanned. - .path("file:///path/to/data") - // schema of ORC files - .forOrcSchema("struct>>") - // Hadoop configuration - .withConfiguration(config) - // build OrcTableSource - .build() -{% endhighlight %} -
    -
    - -**Note:** The `OrcTableSource` does not support ORC's `Union` type yet. - -{% top %} - -Provided TableSinks -------------------- - -The following table lists the `TableSink`s which are provided with Flink. - -| **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description** -| `CsvTableSink` | `flink-table` | Y | Append | A simple sink for CSV files. -| `JDBCAppendTableSink` | `flink-jdbc` | Y | Append | Writes a Table to a JDBC table. -| `CassandraAppendTableSink` | `flink-connector-cassandra` | N | Append | Writes a Table to a Cassandra table. -| `Kafka08JsonTableSink` | `flink-connector-kafka-0.8` | N | Append | A Kafka 0.8 sink with JSON encoding. -| `Kafka09JsonTableSink` | `flink-connector-kafka-0.9` | N | Append | A Kafka 0.9 sink with JSON encoding. -| `Kafka010JsonTableSink` | `flink-connector-kafka-0.10` | N | Append | A Kafka 0.10 sink with JSON encoding. - -All sinks that come with the `flink-table` dependency can be directly used by your Table programs. For all other table sinks, you have to add the respective dependency in addition to the `flink-table` dependency. - -A custom `TableSink` can be defined by implementing the `BatchTableSink`, `AppendStreamTableSink`, `RetractStreamTableSink`, or `UpsertStreamTableSink` interface. See section on [defining a custom TableSink](#define-a-tablesink) for details. - -{% top %} - -### KafkaJsonTableSink - -A `KafkaJsonTableSink` emits a [streaming append `Table`](./streaming.html#table-to-stream-conversion) to an Apache Kafka topic. The rows of the table are encoded as JSON records. Currently, only tables with flat schema, i.e., non-nested fields, are supported. - -A `KafkaJsonTableSink` produces with at-least-once guarantees into a Kafka topic if the query is executed with [checkpointing enabled]({{ site.baseurl }}/dev/stream/state/checkpointing.html#enabling-and-configuring-checkpointing). - -By default, a `KafkaJsonTableSink` writes to at most as many partitions as its own parallelism (each parallel instance of the sink writes to exactly one partition). In order to distribute the writes to more partitions or control the routing of rows into partitions, a custom `FlinkKafkaPartitioner` can be provided. - -The following example shows how to create a `KafkaJsonTableSink` for Kafka 0.10. Sinks for Kafka 0.8 and 0.9 are instantiated analogously. - -
    -
    -{% highlight java %} - -Table table = ... - -Properties props = new Properties(); -props.setProperty("bootstrap.servers", "localhost:9092"); - -table.writeToSink( - new Kafka010JsonTableSink( - "myTopic", // Kafka topic to write to - props)); // Properties to configure the producer - -{% endhighlight %} -
    - -
    -{% highlight scala %} - -val table: Table = ??? - -val props = new Properties() -props.setProperty("bootstrap.servers", "localhost:9092") - -table.writeToSink( - new Kafka010JsonTableSink( - "myTopic", // Kafka topic to write to - props)) // Properties to configure the producer - -{% endhighlight %} -
    -
    - -### CsvTableSink - -The `CsvTableSink` emits a `Table` to one or more CSV files. - -The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming.html#table-to-stream-conversion) for details. When emitting a streaming table, rows are written at least once (if checkpointing is enabled) and the `CsvTableSink` does not split output files into bucket files but continuously writes to the same files. - -
    -
    -{% highlight java %} - -Table table = ... - -table.writeToSink( - new CsvTableSink( - path, // output path - "|", // optional: delimit files by '|' - 1, // optional: write to a single file - WriteMode.OVERWRITE)); // optional: override existing files - -{% endhighlight %} -
    - -
    -{% highlight scala %} - -val table: Table = ??? - -table.writeToSink( - new CsvTableSink( - path, // output path - fieldDelim = "|", // optional: delimit files by '|' - numFiles = 1, // optional: write to a single file - writeMode = WriteMode.OVERWRITE)) // optional: override existing files - -{% endhighlight %} -
    -
    - -### JDBCAppendTableSink - -The `JDBCAppendTableSink` emits a `Table` to a JDBC connection. The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming.html#table-to-stream-conversion) for details. - -The `JDBCAppendTableSink` inserts each `Table` row at least once into the database table (if checkpointing is enabled). However, you can specify the insertion query using REPLACE or INSERT OVERWRITE to perform upsert writes to the database. - -To use the JDBC sink, you have to add the JDBC connector dependency (flink-jdbc) to your project. Then you can create the sink using JDBCAppendSinkBuilder: - -
    -
    -{% highlight java %} - -JDBCAppendTableSink sink = JDBCAppendTableSink.builder() - .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") - .setDBUrl("jdbc:derby:memory:ebookshop") - .setQuery("INSERT INTO books (id) VALUES (?)") - .setParameterTypes(INT_TYPE_INFO) - .build(); - -Table table = ... -table.writeToSink(sink); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val sink: JDBCAppendTableSink = JDBCAppendTableSink.builder() - .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") - .setDBUrl("jdbc:derby:memory:ebookshop") - .setQuery("INSERT INTO books (id) VALUES (?)") - .setParameterTypes(INT_TYPE_INFO) - .build() - -val table: Table = ??? -table.writeToSink(sink) -{% endhighlight %} -
    -
    - -Similar to using JDBCOutputFormat, you have to explicitly specify the name of the JDBC driver, the JDBC URL, the query to be executed, and the field types of the JDBC table. - -{% top %} - -### CassandraAppendTableSink - -The `CassandraAppendTableSink` emits a `Table` to a Cassandra table. The sink only supports append-only streaming tables. It cannot be used to emit a `Table` that is continuously updated. See the [documentation on Table to Stream conversions](./streaming.html#table-to-stream-conversion) for details. - -The `CassandraAppendTableSink` inserts all rows at least once into the Cassandra table if checkpointing is enabled. However, you can specify the query as upsert query. - -To use the `CassandraAppendTableSink`, you have to add the Cassandra connector dependency (flink-connector-cassandra) to your project. The example below shows how to use the `CassandraAppendTableSink`. - -
    -
    -{% highlight java %} - -ClusterBuilder builder = ... // configure Cassandra cluster connection - -CassandraAppendTableSink sink = new CassandraAppendTableSink( - builder, - // the query must match the schema of the table - INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)); - -Table table = ... -table.writeToSink(sink); -{% endhighlight %} -
    - -
    -{% highlight scala %} -val builder: ClusterBuilder = ... // configure Cassandra cluster connection - -val sink: CassandraAppendTableSink = new CassandraAppendTableSink( - builder, - // the query must match the schema of the table - INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)) - -val table: Table = ??? -table.writeToSink(sink) -{% endhighlight %} -
    -
    - -{% top %} - Define a TableSource -------------------- @@ -822,7 +88,7 @@ The `BatchTableSource` interface extends the `TableSource` interface and defines
    {% highlight java %} -BatchTableSource extends TableSource { +BatchTableSource implements TableSource { public DataSet getDataSet(ExecutionEnvironment execEnv); } @@ -850,7 +116,7 @@ The `StreamTableSource` interface extends the `TableSource` interface and define
    {% highlight java %} -StreamTableSource extends TableSource { +StreamTableSource implements TableSource { public DataStream getDataStream(StreamExecutionEnvironment execEnv); } @@ -869,15 +135,17 @@ StreamTableSource[T] extends TableSource[T] { * `getDataStream(execEnv)`: Returns a `DataStream` with the data of the table. The type of the `DataStream` must be identical to the return type defined by the `TableSource.getReturnType()` method. The `DataStream` can by created using a regular [data source]({{ site.baseurl }}/dev/datastream_api.html#data-sources) of the DataStream API. Commonly, a `StreamTableSource` is implemented by wrapping a `SourceFunction` or a [stream connector]({{ site.baseurl }}/dev/connectors/). +{% top %} + ### Defining a TableSource with Time Attributes -Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL](sql.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes]({{ site.baseurl }}/dev/table/streaming.html#time-attributes). +Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL](sql.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes](streaming/time_attributes.html). A `TableSource` defines a time attribute as a field of type `Types.SQL_TIMESTAMP` in its table schema. In contrast to all regular fields in the schema, a time attribute must not be matched to a physical field in the return type of the table source. Instead, a `TableSource` defines a time attribute by implementing a certain interface. #### Defining a Processing Time Attribute -A `TableSource` defines a [processing time attribute](streaming.html#processing-time) by implementing the `DefinedProctimeAttribute` interface. The interface looks as follows: +[Processing time attributes](streaming/time_attributes.html#processing-time) are commonly used in streaming queries. A processing time attribute returns the current wall-clock time of the operator that accesses it. A `TableSource` defines a processing time attribute by implementing the `DefinedProctimeAttribute` interface. The interface looks as follows:
    @@ -901,11 +169,19 @@ DefinedProctimeAttribute { * `getProctimeAttribute()`: Returns the name of the processing time attribute. The specified attribute must be defined of type `Types.SQL_TIMESTAMP` in the table schema and can be used in time-based operations. A `DefinedProctimeAttribute` table source can define no processing time attribute by returning `null`. -**Note** Both `StreamTableSource` and `BatchTableSource` can implement `DefinedProctimeAttribute` and define a processing time attribute. In case of a `BatchTableSource` the processing time field is initialized with the current timestamp during the table scan. +Attention Both `StreamTableSource` and `BatchTableSource` can implement `DefinedProctimeAttribute` and define a processing time attribute. In case of a `BatchTableSource` the processing time field is initialized with the current timestamp during the table scan. #### Defining a Rowtime Attribute -A `TableSource` defines a [rowtime attribute](streaming.html#event-time) by implementing the `DefinedRowtimeAttributes` interface. The interface looks as follows: +[Rowtime attributes](streaming/time_attributes.html#event-time) are attributes of type `TIMESTAMP` and handled in a unified way in stream and batch queries. + +A table schema field of type `SQL_TIMESTAMP` can be declared as rowtime attribute by specifying + +* the name of the field, +* a `TimestampExtractor` that computes the actual value for the attribute (usually from one or more other fields), and +* a `WatermarkStrategy` that specifies how watermarks are generated for the the rowtime attribute. + +A `TableSource` defines a rowtime attribute by implementing the `DefinedRowtimeAttributes` interface. The interface looks as follows:
    @@ -929,11 +205,35 @@ DefinedRowtimeAttributes { * `getRowtimeAttributeDescriptors()`: Returns a list of `RowtimeAttributeDescriptor`. A `RowtimeAttributeDescriptor` describes a rowtime attribute with the following properties: * `attributeName`: The name of the rowtime attribute in the table schema. The field must be defined with type `Types.SQL_TIMESTAMP`. - * `timestampExtractor`: The timestamp extractor extracts the timestamp from a record with the return type. For example, it can convert convert a Long field into a timestamp or parse a String-encoded timestamp. Flink comes with a set of built-in `TimestampExtractor` implementation for common use cases. It is also possible to provide a custom implementation. + * `timestampExtractor`: The timestamp extractor extracts the timestamp from a record with the return type. For example, it can convert a Long field into a timestamp or parse a String-encoded timestamp. Flink comes with a set of built-in `TimestampExtractor` implementation for common use cases. It is also possible to provide a custom implementation. * `watermarkStrategy`: The watermark strategy defines how watermarks are generated for the rowtime attribute. Flink comes with a set of built-in `WatermarkStrategy` implementations for common use cases. It is also possible to provide a custom implementation. -* **Note** Although the `getRowtimeAttributeDescriptors()` method returns a list of descriptors, only a single rowtime attribute is support at the moment. We plan to remove this restriction in the future and support tables with more than one rowtime attribute. -**IMPORTANT** Both, `StreamTableSource` and `BatchTableSource`, can implement `DefinedRowtimeAttributes` and define a rowtime attribute. In either case, the rowtime field is extracted using the `TimestampExtractor`. Hence, a `TableSource` that implements `StreamTableSource` and `BatchTableSource` and defines a rowtime attribute provides exactly the same data to streaming and batch queries. +Attention Although the `getRowtimeAttributeDescriptors()` method returns a list of descriptors, only a single rowtime attribute is support at the moment. We plan to remove this restriction in the future and support tables with more than one rowtime attribute. + +Attention Both, `StreamTableSource` and `BatchTableSource`, can implement `DefinedRowtimeAttributes` and define a rowtime attribute. In either case, the rowtime field is extracted using the `TimestampExtractor`. Hence, a `TableSource` that implements `StreamTableSource` and `BatchTableSource` and defines a rowtime attribute provides exactly the same data to streaming and batch queries. + +##### Provided Timestamp Extractors + +Flink provides `TimestampExtractor` implementations for common use cases. + +The following `TimestampExtractor` implementations are currently available: + +* `ExistingField(fieldName)`: Extracts the value of a rowtime attribute from an existing `LONG`, `SQL_TIMESTAMP`, or timestamp formatted `STRING` field. One example of such a string would be '2018-05-28 12:34:56.000'. +* `StreamRecordTimestamp()`: Extracts the value of a rowtime attribute from the timestamp of the `DataStream` `StreamRecord`. Note, this `TimestampExtractor` is not available for batch table sources. + +A custom `TimestampExtractor` can be defined by implementing the corresponding interface. + +##### Provided Watermark Strategies + +Flink provides `WatermarkStrategy` implementations for common use cases. + +The following `WatermarkStrategy` implementations are currently available: + +* `AscendingTimestamps`: A watermark strategy for ascending timestamps. Records with timestamps that are out-of-order will be considered late. +* `BoundedOutOfOrderTimestamps(delay)`: A watermark strategy for timestamps that are at most out-of-order by the specified delay. +* `PreserveWatermarks()`: A strategy which indicates the watermarks should be preserved from the underlying `DataStream`. + +A custom `WatermarkStrategy` can be defined by implementing the corresponding interface. {% top %} @@ -1074,7 +374,7 @@ The interface looks as follows:
    {% highlight java %} -BatchTableSink extends TableSink { +BatchTableSink implements TableSink { public void emitDataSet(DataSet dataSet); } @@ -1102,7 +402,7 @@ The interface looks as follows:
    {% highlight java %} -AppendStreamTableSink extends TableSink { +AppendStreamTableSink implements TableSink { public void emitDataStream(DataStream dataStream); } @@ -1132,7 +432,7 @@ The interface looks as follows:
    {% highlight java %} -RetractStreamTableSink extends TableSink> { +RetractStreamTableSink implements TableSink> { public TypeInformation getRecordType(); @@ -1166,7 +466,7 @@ The interface looks as follows:
    {% highlight java %} -UpsertStreamTableSink extends TableSink> { +UpsertStreamTableSink implements TableSink> { public void setKeyFields(String[] keys); @@ -1203,3 +503,245 @@ A message with true boolean field is an upsert message for the configured key. A {% top %} +Define a TableFactory +--------------------- + +A `TableFactory` allows to create different table-related instances from string-based properties. All available factories are called for matching to the given set of properties and a corresponding factory class. + +Factories leverage Java's [Service Provider Interfaces (SPI)](https://docs.oracle.com/javase/tutorial/sound/SPI-intro.html) for discovering. This means that every dependency and JAR file should contain a file `org.apache.flink.table.factories.TableFactory` in the `META_INF/services` resource directory that lists all available table factories that it provides. + +Every table factory needs to implement the following interface: + +
    +
    +{% highlight java %} +package org.apache.flink.table.factories; + +interface TableFactory { + + Map requiredContext(); + + List supportedProperties(); +} +{% endhighlight %} +
    + +
    +{% highlight scala %} +package org.apache.flink.table.factories + +trait TableFactory { + + def requiredContext(): util.Map[String, String] + + def supportedProperties(): util.List[String] +} +{% endhighlight %} +
    +
    + +* `requiredContext()`: Specifies the context that this factory has been implemented for. The framework guarantees to only match for this factory if the specified set of properties and values are met. Typical properties might be `connector.type`, `format.type`, or `update-mode`. Property keys such as `connector.property-version` and `format.property-version` are reserved for future backwards compatibility cases. +* `supportedProperties`: List of property keys that this factory can handle. This method will be used for validation. If a property is passed that this factory cannot handle, an exception will be thrown. The list must not contain the keys that are specified by the context. + +In order to create a specific instance, a factory class can implement one or more interfaces provided in `org.apache.flink.table.factories`: + +* `BatchTableSourceFactory`: Creates a batch table source. +* `BatchTableSinkFactory`: Creates a batch table sink. +* `StreamTableSourceFactory`: Creates a stream table source. +* `StreamTableSinkFactory`: Creates a stream table sink. +* `DeserializationSchemaFactory`: Creates a deserialization schema format. +* `SerializationSchemaFactory`: Creates a serialization schema format. + +The discovery of a factory happens in multiple stages: + +- Discover all available factories. +- Filter by factory class (e.g., `StreamTableSourceFactory`). +- Filter by matching context. +- Filter by supported properties. +- Verify that exactly one factory matches, otherwise throw an `AmbiguousTableFactoryException` or `NoMatchingTableFactoryException`. + +The following example shows how to provide a custom streaming source with an additional `connector.debug` property flag for parameterization. + +
    +
    +{% highlight java %} +import org.apache.flink.table.sources.StreamTableSource; +import org.apache.flink.types.Row; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class MySystemTableSourceFactory implements StreamTableSourceFactory { + + @Override + public Map requiredContext() { + Map context = new HashMap<>(); + context.put("update-mode", "append"); + context.put("connector.type", "my-system"); + return context; + } + + @Override + public List supportedProperties() { + List list = new ArrayList<>(); + list.add("connector.debug"); + return list; + } + + @Override + public StreamTableSource createStreamTableSource(Map properties) { + boolean isDebug = Boolean.valueOf(properties.get("connector.debug")); + + # additional validation of the passed properties can also happen here + + return new MySystemAppendTableSource(isDebug); + } +} +{% endhighlight %} +
    + +
    +{% highlight scala %} +import java.util +import org.apache.flink.table.sources.StreamTableSource +import org.apache.flink.types.Row + +class MySystemTableSourceFactory extends StreamTableSourceFactory[Row] { + + override def requiredContext(): util.Map[String, String] = { + val context = new util.HashMap[String, String]() + context.put("update-mode", "append") + context.put("connector.type", "my-system") + context + } + + override def supportedProperties(): util.List[String] = { + val properties = new util.ArrayList[String]() + properties.add("connector.debug") + properties + } + + override def createStreamTableSource(properties: util.Map[String, String]): StreamTableSource[Row] = { + val isDebug = java.lang.Boolean.valueOf(properties.get("connector.debug")) + + # additional validation of the passed properties can also happen here + + new MySystemAppendTableSource(isDebug) + } +} +{% endhighlight %} +
    +
    + +{% top %} + +### Use a TableFactory in the SQL Client + +In a SQL Client environment file, the previously presented factory could be declared as: + +{% highlight yaml %} +tables: + - name: MySystemTable + type: source + update-mode: append + connector: + type: my-system + debug: true +{% endhighlight %} + +The YAML file is translated into flattened string properties and a table factory is called with those properties that describe the connection to the external system: + +{% highlight text %} +update-mode=append +connector.type=my-system +connector.debug=true +{% endhighlight %} + +Attention Properties such as `tables.#.name` or `tables.#.type` are SQL Client specifics and are not passed to any factory. The `type` property decides, depending on the execution environment, whether a `BatchTableSourceFactory`/`StreamTableSourceFactory` (for `source`), a `BatchTableSinkFactory`/`StreamTableSinkFactory` (for `sink`), or both (for `both`) need to discovered. + +{% top %} + +### Use a TableFactory in the Table & SQL API + +For a type-safe, programmatic approach with explanatory Scaladoc/Javadoc, the Table & SQL API offers descriptors in `org.apache.flink.table.descriptors` that translate into string-based properties. See the [built-in descriptors](connect.html) for sources, sinks, and formats as a reference. + +A connector for `MySystem` in our example can extend `ConnectorDescriptor` as shown below: + +
    +
    +{% highlight java %} +import org.apache.flink.table.descriptors.ConnectorDescriptor; +import java.util.HashMap; +import java.util.Map; + +/** + * Connector to MySystem with debug mode. + */ +public class MySystemConnector extends ConnectorDescriptor { + + public final boolean isDebug; + + public MySystemConnector(boolean isDebug) { + super("my-system", 1, false); + this.isDebug = isDebug; + } + + @Override + protected Map toConnectorProperties() { + Map properties = new HashMap<>(); + properties.put("connector.debug", Boolean.toString(isDebug)); + return properties; + } +} +{% endhighlight %} +
    + +
    +{% highlight scala %} +import org.apache.flink.table.descriptors.ConnectorDescriptor +import java.util.HashMap +import java.util.Map + +/** + * Connector to MySystem with debug mode. + */ +class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system", 1, false) { + + override protected def toConnectorProperties(): Map[String, String] = { + val properties = new HashMap[String, String] + properties.put("connector.debug", isDebug.toString) + properties + } +} +{% endhighlight %} +
    +
    + +The descriptor can then be used in the API as follows: + +
    +
    +{% highlight java %} +StreamTableEnvironment tableEnv = // ... + +tableEnv + .connect(new MySystemConnector(true)) + .inAppendMode() + .registerTableSource("MySystemTable"); +{% endhighlight %} +
    + +
    +{% highlight scala %} +val tableEnv: StreamTableEnvironment = // ... + +tableEnv + .connect(new MySystemConnector(isDebug = true)) + .inAppendMode() + .registerTableSource("MySystemTable") +{% endhighlight %} +
    +
    + +{% top %} diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql.md index eb7204a4f13103..14df44131f9621 100644 --- a/docs/dev/table/sql.md +++ b/docs/dev/table/sql.md @@ -22,9 +22,9 @@ specific language governing permissions and limitations under the License. --> -SQL queries are specified with the `sql()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can seamlessly mixed and are holistically optimized and translated into a single program. +SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program. -In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-a-table-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources. +In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources. For convenience `Table.toString()` automatically registers the table under a unique name in its `TableEnvironment` and returns the name. Hence, `Table` objects can be directly inlined into SQL queries (by string concatenation) as shown in the examples below. @@ -48,7 +48,7 @@ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); DataStream> ds = env.addSource(...); // SQL query with an inlined (unregistered) table -Table table = tableEnv.toTable(ds, "user, product, amount"); +Table table = tableEnv.fromDataStream(ds, "user, product, amount"); Table result = tableEnv.sqlQuery( "SELECT SUM(amount) FROM " + table + " WHERE product LIKE '%Rubber%'"); @@ -93,7 +93,7 @@ val result2 = tableEnv.sqlQuery( // SQL update with a registered table // create and register a TableSink -TableSink csvSink = new CsvTableSink("/path/to/file", ...) +val csvSink: CsvTableSink = new CsvTableSink("/path/to/file", ...) val fieldNames: Array[String] = Array("product", "amount") val fieldTypes: Array[TypeInformation[_]] = Array(Types.STRING, Types.INT) tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink) @@ -113,8 +113,12 @@ Flink parses SQL using [Apache Calcite](https://calcite.apache.org/docs/referenc The following BNF-grammar describes the superset of supported SQL features in batch and streaming queries. The [Operations](#operations) section shows examples for the supported features and indicates which features are only supported for batch or streaming queries. -``` +{% highlight sql %} +insert: + INSERT INTO tableReference + query + query: values | { @@ -139,7 +143,8 @@ select: [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ] - + [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] + selectWithoutFrom: SELECT [ ALL | DISTINCT ] { * | projectItem [, projectItem ]* } @@ -158,6 +163,7 @@ joinCondition: tableReference: tablePrimary + [ matchRecognize ] [ [ AS ] alias [ '(' columnAlias [, columnAlias ]* ')' ] ] tablePrimary: @@ -176,11 +182,62 @@ groupItem: | ROLLUP '(' expression [, expression ]* ')' | GROUPING SETS '(' groupItem [, groupItem ]* ')' -insert: - INSERT INTO tableReference - query +windowRef: + windowName + | windowSpec -``` +windowSpec: + [ windowName ] + '(' + [ ORDER BY orderItem [, orderItem ]* ] + [ PARTITION BY expression [, expression ]* ] + [ + RANGE numericOrIntervalExpression {PRECEDING} + | ROWS numericExpression {PRECEDING} + ] + ')' + +matchRecognize: + MATCH_RECOGNIZE '(' + [ PARTITION BY expression [, expression ]* ] + [ ORDER BY orderItem [, orderItem ]* ] + [ MEASURES measureColumn [, measureColumn ]* ] + [ ONE ROW PER MATCH ] + [ AFTER MATCH + ( SKIP TO NEXT ROW + | SKIP PAST LAST ROW + | SKIP TO FIRST variable + | SKIP TO LAST variable + | SKIP TO variable ) + ] + PATTERN '(' pattern ')' + [ WITHIN intervalLiteral ] + DEFINE variable AS condition [, variable AS condition ]* + ')' + +measureColumn: + expression AS alias + +pattern: + patternTerm [ '|' patternTerm ]* + +patternTerm: + patternFactor [ patternFactor ]* + +patternFactor: + variable [ patternQuantifier ] + +patternQuantifier: + '*' + | '*?' + | '+' + | '+?' + | '?' + | '??' + | '{' { [ minRepeat ], [ maxRepeat ] } '}' ['?'] + | '{' repeat '}' + +{% endhighlight %} Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java: @@ -188,6 +245,11 @@ Flink SQL uses a lexical policy for identifier (table, attribute, function names - After which, identifiers are matched case-sensitively. - Unlike Java, back-ticks allow identifiers to contain non-alphanumeric characters (e.g. "SELECT a AS `my field` FROM t"). +String literals must be enclosed in single quotes (e.g., `SELECT 'Hello World'`). Duplicate a single quote for escaping (e.g., `SELECT 'It''s me.'`). Unicode characters are supported in string literals. If explicit unicode code points are required, use the following syntax: + +- Use the backslash (`\`) as escaping character (default): `SELECT U&'\263A'` +- Use a custom escaping character: `SELECT U&'#263A' UESCAPE '#'` + {% top %} Operations @@ -266,7 +328,7 @@ SELECT PRETTY_PRINT(user) FROM Orders Result Updating -

    Note: GroupBy on a streaming table produces an updating result. See the Streaming Concepts page for details. +

    Note: GroupBy on a streaming table produces an updating result. See the Dynamic Tables Streaming Concepts page for details.

    {% highlight sql %} SELECT a, SUM(b) as d @@ -295,13 +357,20 @@ GROUP BY TUMBLE(rowtime, INTERVAL '1' DAY), user Streaming -

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute

    +

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute

    {% highlight sql %} SELECT COUNT(amount) OVER ( PARTITION BY user ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM Orders + +SELECT COUNT(amount) OVER w, SUM(amount) OVER w +FROM Orders +WINDOW w AS ( + PARTITION BY user + ORDER BY proctime + ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) {% endhighlight %} @@ -315,7 +384,7 @@ FROM Orders {% highlight sql %} SELECT DISTINCT users FROM Orders {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -388,12 +457,14 @@ GROUP BY users SELECT * FROM Orders INNER JOIN Product ON Orders.productId = Product.id {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    Outer Equi-join
    Batch + Streaming + Result Updating

    Currently, only equi-joins are supported, i.e., joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or theta joins are not supported.

    @@ -408,6 +479,7 @@ FROM Orders RIGHT JOIN Product ON Orders.productId = Product.id SELECT * FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id {% endhighlight %} +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -418,9 +490,9 @@ FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id

    Note: Time-windowed joins are a subset of regular joins that can be processed in a streaming fashion.

    -

    A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >), a BETWEEN predicate, or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

    +

    A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >), a BETWEEN predicate, or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

    For example, the following predicates are valid window join conditions:

    - +
    • ltime = rtime
    • ltime >= rtime AND ltime < rtime + INTERVAL '10' MINUTE
      • @@ -452,25 +524,56 @@ FROM Orders CROSS JOIN UNNEST(tags) AS t (tag) - Join with User Defined Table Functions (UDTF)
      + Join with Table Function
      Batch Streaming -

      UDTFs must be registered in the TableEnvironment. See the UDF documentation for details on how to specify and register UDTFs.

      -

      Inner Join

      +

      Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function.

      +

      User-defined table functions (UDTFs) must be registered before. See the UDF documentation for details on how to specify and register UDTFs.

      + +

      Inner Join

      +

      A row of the left (outer) table is dropped, if its table function call returns an empty result.

      {% highlight sql %} SELECT users, tag FROM Orders, LATERAL TABLE(unnest_udtf(tags)) t AS tag {% endhighlight %} -

      Left Outer Join

      + +

      Left Outer Join

      +

      If a table function call returns an empty result, the corresponding outer row is preserved and the result padded with null values.

      {% highlight sql %} SELECT users, tag FROM Orders LEFT JOIN LATERAL TABLE(unnest_udtf(tags)) t AS tag ON TRUE {% endhighlight %} -

      Note: Currently, only literal TRUE is supported as predicate for a left outer join against a lateral table.

      +

      Note: Currently, only literal TRUE is supported as predicate for a left outer join against a lateral table.

      + + + + + Join with Temporal Table
      + Streaming + + +

      Temporal tables are tables that track changes over time.

      +

      A Temporal table function provides access to the state of a temporal table at a specific point in time. + The syntax to join a table with a temporal table function is the same as in Join with Table Function.

      + +

      Note: Currently only inner joins with temporal tables are supported.

      + +

      Assuming Rates is a temporal table function, the join can be expressed in SQL as follows:

      +{% highlight sql %} +SELECT + o_amount, r_rate +FROM + Orders, + LATERAL TABLE (Rates(o_proctime)) +WHERE + r_currency = o_currency +{% endhighlight %} +

      For more information please check the more detailed temporal tables concept description.

      +
    @@ -549,10 +652,10 @@ FROM ( In
    - Batch + Batch Streaming - Returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression. +

    Returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression.

    {% highlight sql %} SELECT user, amount FROM Orders @@ -560,6 +663,25 @@ WHERE product IN ( SELECT product FROM NewProducts ) {% endhighlight %} +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    + + + + + + Exists
    + Batch Streaming + + +

    Returns true if the sub-query returns at least one row. Only supported if the operation can be rewritten in a join and group operation.

    +{% highlight sql %} +SELECT user, amount +FROM Orders +WHERE product EXISTS ( + SELECT product FROM NewProducts +) +{% endhighlight %} +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -585,7 +707,7 @@ WHERE product IN ( Batch Streaming -Note: The result of streaming queries must be primarily sorted on an ascending time attribute. Additional sorting attributes are supported. +Note: The result of streaming queries must be primarily sorted on an ascending time attribute. Additional sorting attributes are supported. {% highlight sql %} SELECT * @@ -675,10 +797,9 @@ Group windows are defined in the `GROUP BY` clause of a SQL query. Just like que - #### Time Attributes -For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes](streaming.html#time-attributes) to learn how to define time attributes. +For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes](streaming/time_attributes.html) to learn how to define time attributes. For SQL on batch tables, the `time_attr` argument of the group window function must be an attribute of type `TIMESTAMP`. @@ -710,7 +831,7 @@ The start and end timestamps of group windows as well as time attributes can be SESSION_END(time_attr, interval)

    Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.

    -

    Note: The exclusive upper bound timestamp cannot be used as a rowtime attribute in subsequent time-based operations, such as time-windowed joins and group window or over window aggregations.

    +

    Note: The exclusive upper bound timestamp cannot be used as a rowtime attribute in subsequent time-based operations, such as time-windowed joins and group window or over window aggregations.

    @@ -719,7 +840,7 @@ The start and end timestamps of group windows as well as time attributes can be SESSION_ROWTIME(time_attr, interval)

    Returns the timestamp of the inclusive upper bound of the corresponding tumbling, hopping, or session window.

    -

    The resulting attribute is a rowtime attribute that can be used in subsequent time-based operations such as time-windowed joins and group window or over window aggregations.

    +

    The resulting attribute is a rowtime attribute that can be used in subsequent time-based operations such as time-windowed joins and group window or over window aggregations.

    @@ -727,7 +848,7 @@ The start and end timestamps of group windows as well as time attributes can be HOP_PROCTIME(time_attr, interval, interval)
    SESSION_PROCTIME(time_attr, interval)
    -

    Returns a proctime attribute that can be used in subsequent time-based operations such as time-windowed joins and group window or over window aggregations.

    +

    Returns a proctime attribute that can be used in subsequent time-based operations such as time-windowed joins and group window or over window aggregations.

    @@ -821,6 +942,52 @@ val result4 = tableEnv.sqlQuery( {% top %} +### Pattern Recognition + +
    + + + + + + + + + + + + + + +
    OperationDescription
    + MATCH_RECOGNIZE
    + Streaming +     		+

    Searches for a given pattern in a streaming table according to the MATCH_RECOGNIZE ISO standard. This makes it possible to express complex event processing (CEP) logic in SQL queries.

    +

    For a more detailed description, see the dedicated page for detecting patterns in tables.

    + +{% highlight sql %} +SELECT T.aid, T.bid, T.cid +FROM MyTable +MATCH_RECOGNIZE ( + PARTITION BY userid + ORDER BY proctime + MEASURES + A.id AS aid, + B.id AS bid, + C.id AS cid + PATTERN (A B C) + DEFINE + A AS name = 'a', + B AS name = 'b', + C AS name = 'c' +) AS T +{% endhighlight %} +
    +
    + +{% top %} + Data Types ---------- @@ -846,1741 +1013,26 @@ The SQL runtime is built on top of Flink's DataSet and DataStream APIs. Internal | `Types.OBJECT_ARRAY` | `ARRAY` | e.g. `java.lang.Byte[]`| | `Types.MAP` | `MAP` | `java.util.HashMap` | | `Types.MULTISET` | `MULTISET` | e.g. `java.util.HashMap` for a multiset of `String` | +| `Types.ROW` | `ROW` | `org.apache.flink.types.Row` | -Generic types and composite types (e.g., POJOs or Tuples) can be fields of a row as well. Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html). Composite types can be accessed with [built-in functions](#built-in-functions) (see *Value access functions* section). - -{% top %} - -Built-In Functions ------------------- - -Flink's SQL support comes with a set of built-in functions for data transformations. This section gives a brief overview of the available functions. - - - -The Flink SQL functions (including their syntax) are a subset of Apache Calcite's built-in functions. Most of the documentation has been adopted from the [Calcite SQL reference](https://calcite.apache.org/docs/reference.html). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Generic types and (nested) composite types (e.g., POJOs, tuples, rows, Scala case classes) can be fields of a row as well. - - - - +Fields of composite types with arbitrary nesting can be accessed with [value access functions](functions.html#value-access-functions). +Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html). - - - - +{% top %} +Reserved Keywords +----------------- - - - - +Although not every SQL feature is implemented yet, some string combinations are already reserved as keywords for future use. If you want to use one of the following strings as a field name, make sure to surround them with backticks (e.g. `` `value` ``, `` `count` ``). - - - - +{% highlight sql %} - - - - +A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE - - - - - - - - - - - -
    Comparison functionsDescription
    - {% highlight text %} -value1 = value2 -{% endhighlight %} - -

    Equals.

    -
    - {% highlight text %} -value1 <> value2 -{% endhighlight %} - -

    Not equal.

    -
    - {% highlight text %} -value1 > value2 -{% endhighlight %} - -

    Greater than.

    -
    - {% highlight text %} -value1 >= value2 -{% endhighlight %} - -

    Greater than or equal.

    -
    - {% highlight text %} -value1 < value2 -{% endhighlight %} - -

    Less than.

    -
    - {% highlight text %} -value1 <= value2 -{% endhighlight %} - -

    Less than or equal.

    -
    - {% highlight text %} -value IS NULL -{% endhighlight %} - -

    Returns TRUE if value is null.

    -
    - {% highlight text %} -value IS NOT NULL -{% endhighlight %} - -

    Returns TRUE if value is not null.

    -
    - {% highlight text %} -value1 IS DISTINCT FROM value2 -{% endhighlight %} - -

    Returns TRUE if two values are not equal, treating null values as the same.

    -
    - {% highlight text %} -value1 IS NOT DISTINCT FROM value2 -{% endhighlight %} - -

    Returns TRUE if two values are equal, treating null values as the same.

    -
    - {% highlight text %} -value1 BETWEEN [ASYMMETRIC | SYMMETRIC] value2 AND value3 -{% endhighlight %} - -

    Returns TRUE if value1 is greater than or equal to value2 and less than or equal to value3.

    -
    - {% highlight text %} -value1 NOT BETWEEN value2 AND value3 -{% endhighlight %} - -

    Returns TRUE if value1 is less than value2 or greater than value3.

    -
    - {% highlight text %} -string1 LIKE string2 [ ESCAPE string3 ] -{% endhighlight %} - -

    Returns TRUE if string1 matches pattern string2. An escape character can be defined if necessary.

    -
    - {% highlight text %} -string1 NOT LIKE string2 [ ESCAPE string3 ] -{% endhighlight %} - -

    Returns TRUE if string1 does not match pattern string2. An escape character can be defined if necessary.

    -
    - {% highlight text %} -string1 SIMILAR TO string2 [ ESCAPE string3 ] -{% endhighlight %} - -

    Returns TRUE if string1 matches regular expression string2. An escape character can be defined if necessary.

    -
    - {% highlight text %} -string1 NOT SIMILAR TO string2 [ ESCAPE string3 ] -{% endhighlight %} - -

    Returns TRUE if string1 does not match regular expression string2. An escape character can be defined if necessary.

    -
    - {% highlight text %} -value IN (value [, value]* ) -{% endhighlight %} - -

    Returns TRUE if an expression exists in a given list of expressions. This is a shorthand for multiple OR conditions. If the testing set contains NULL, the result will be NULL if the element can not be found and TRUE if it can be found. If the element is NULL, the result is always NULL. E.g. "42 IN (1, 2, 3)" leads to FALSE.

    -
    - {% highlight text %} -value NOT IN (value [, value]* ) -{% endhighlight %} - -

    Returns TRUE if value is not equal to every value in a list.

    -
    - {% highlight text %} -EXISTS (sub-query) -{% endhighlight %} - -

    Returns TRUE if sub-query returns at least one row. Only supported if the operation can be rewritten in a join and group operation.

    -
    - {% highlight text %} -value IN (sub-query) {% endhighlight %} - -

    Returns TRUE if value is equal to a row returned by sub-query. This operation is not supported in a streaming environment yet.

    -
    - {% highlight text %} -value NOT IN (sub-query) -{% endhighlight %} - -

    Returns TRUE if value is not equal to every row returned by sub-query. This operation is not supported in a streaming environment yet.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Logical functionsDescription
    - {% highlight text %} -boolean1 OR boolean2 -{% endhighlight %} - -

    Returns TRUE if boolean1 is TRUE or boolean2 is TRUE. Supports three-valued logic.

    -
    - {% highlight text %} -boolean1 AND boolean2 -{% endhighlight %} - -

    Returns TRUE if boolean1 and boolean2 are both TRUE. Supports three-valued logic.

    -
    - {% highlight text %} -NOT boolean -{% endhighlight %} - -

    Returns TRUE if boolean is not TRUE; returns UNKNOWN if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS FALSE -{% endhighlight %} - -

    Returns TRUE if boolean is FALSE; returns FALSE if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS NOT FALSE -{% endhighlight %} - -

    Returns TRUE if boolean is not FALSE; returns TRUE if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS TRUE -{% endhighlight %} - -

    Returns TRUE if boolean is TRUE; returns FALSE if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS NOT TRUE -{% endhighlight %} - -

    Returns TRUE if boolean is not TRUE; returns TRUE if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS UNKNOWN -{% endhighlight %} - -

    Returns TRUE if boolean is UNKNOWN.

    -
    - {% highlight text %} -boolean IS NOT UNKNOWN -{% endhighlight %} - -

    Returns TRUE if boolean is not UNKNOWN.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Arithmetic functionsDescription
    - {% highlight text %} -+ numeric -{% endhighlight %} - -

    Returns numeric.

    -
    - {% highlight text %} -- numeric -{% endhighlight %} - -

    Returns negative numeric.

    -
    - {% highlight text %} -numeric1 + numeric2 -{% endhighlight %} - -

    Returns numeric1 plus numeric2.

    -
    - {% highlight text %} -numeric1 - numeric2 -{% endhighlight %} - -

    Returns numeric1 minus numeric2.

    -
    - {% highlight text %} -numeric1 * numeric2 -{% endhighlight %} - -

    Returns numeric1 multiplied by numeric2.

    -
    - {% highlight text %} -numeric1 / numeric2 -{% endhighlight %} - -

    Returns numeric1 divided by numeric2.

    -
    - {% highlight text %} -POWER(numeric1, numeric2) -{% endhighlight %} - -

    Returns numeric1 raised to the power of numeric2.

    -
    - {% highlight text %} -ABS(numeric) -{% endhighlight %} - -

    Returns the absolute value of numeric.

    -
    - {% highlight text %} -MOD(numeric1, numeric2) -{% endhighlight %} - -

    Returns the remainder (modulus) of numeric1 divided by numeric2. The result is negative only if numeric1 is negative.

    -
    - {% highlight text %} -SQRT(numeric) -{% endhighlight %} - -

    Returns the square root of numeric.

    -
    - {% highlight text %} -LN(numeric) -{% endhighlight %} - -

    Returns the natural logarithm (base e) of numeric.

    -
    - {% highlight text %} -LOG10(numeric) -{% endhighlight %} - -

    Returns the base 10 logarithm of numeric.

    -
    - {% highlight text %} -EXP(numeric) -{% endhighlight %} - -

    Returns e raised to the power of numeric.

    -
    - {% highlight text %} -CEIL(numeric) -CEILING(numeric) -{% endhighlight %} - -

    Rounds numeric up, and returns the smallest number that is greater than or equal to numeric.

    -
    - {% highlight text %} -FLOOR(numeric) -{% endhighlight %} - -

    Rounds numeric down, and returns the largest number that is less than or equal to numeric.

    -
    - {% highlight text %} -SIN(numeric) -{% endhighlight %} - -

    Calculates the sine of a given number.

    -
    - {% highlight text %} -COS(numeric) -{% endhighlight %} - -

    Calculates the cosine of a given number.

    -
    - {% highlight text %} -TAN(numeric) -{% endhighlight %} - -

    Calculates the tangent of a given number.

    -
    - {% highlight text %} -COT(numeric) -{% endhighlight %} - -

    Calculates the cotangent of a given number.

    -
    - {% highlight text %} -ASIN(numeric) -{% endhighlight %} - -

    Calculates the arc sine of a given number.

    -
    - {% highlight text %} -ACOS(numeric) -{% endhighlight %} - -

    Calculates the arc cosine of a given number.

    -
    - {% highlight text %} -ATAN(numeric) -{% endhighlight %} - -

    Calculates the arc tangent of a given number.

    -
    - {% highlight text %} -DEGREES(numeric) -{% endhighlight %} - -

    Converts numeric from radians to degrees.

    -
    - {% highlight text %} -RADIANS(numeric) -{% endhighlight %} - -

    Converts numeric from degrees to radians.

    -
    - {% highlight text %} -SIGN(numeric) -{% endhighlight %} - -

    Calculates the signum of a given number.

    -
    - {% highlight text %} -ROUND(numeric, int) -{% endhighlight %} - -

    Rounds the given number to integer places right to the decimal point.

    -
    - {% highlight text %} -PI() -{% endhighlight %} - -

    Returns a value that is closer than any other value to pi.

    -
    - {% highlight text %} -E() -{% endhighlight %} - -

    Returns a value that is closer than any other value to e.

    -
    - {% highlight text %} -RAND() -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    -
    - {% highlight text %} -RAND(seed integer) -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with a initial seed. Two RAND functions will return identical sequences of numbers if they have same initial seed.

    -
    - {% highlight text %} -RAND_INTEGER(bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive).

    -
    - {% highlight text %} -RAND_INTEGER(seed integer, bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive) with a initial seed. Two RAND_INTEGER functions will return identical sequences of numbers if they have same initial seed and same bound.

    -
    - {% highlight text %} -LOG(x numeric), LOG(base numeric, x numeric) -{% endhighlight %} - -

    Returns the natural logarithm of a specified number of a specified base. If called with one parameter, this function returns the natural logarithm of x. If called with two parameters, this function returns the logarithm of x to the base b. x must be greater than 0. b must be greater than 1.

    -
    -{% highlight text %} -BIN(numeric) - {% endhighlight %} - -

    Returns a string representation of an integer numeric value in binary format. Returns null if numeric is null. E.g. "4" leads to "100", "12" leads to "1100".

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    String functionsDescription
    - {% highlight text %} -string || string -{% endhighlight %} - -

    Concatenates two character strings.

    -
    - {% highlight text %} -CHAR_LENGTH(string) -{% endhighlight %} - -

    Returns the number of characters in a character string.

    -
    - {% highlight text %} -CHARACTER_LENGTH(string) -{% endhighlight %} - -

    As CHAR_LENGTH(string).

    -
    - {% highlight text %} -UPPER(string) -{% endhighlight %} - -

    Returns a character string converted to upper case.

    -
    - {% highlight text %} -LOWER(string) -{% endhighlight %} - -

    Returns a character string converted to lower case.

    -
    - {% highlight text %} -POSITION(string1 IN string2) -{% endhighlight %} - -

    Returns the position of the first occurrence of string1 in string2.

    -
    - {% highlight text %} -TRIM( { BOTH | LEADING | TRAILING } string1 FROM string2) -{% endhighlight %} - -

    Removes leading and/or trailing characters from string2. By default, whitespaces at both sides are removed.

    -
    - {% highlight text %} -OVERLAY(string1 PLACING string2 FROM integer [ FOR integer2 ]) -{% endhighlight %} - -

    Replaces a substring of string1 with string2.

    -
    - {% highlight text %} -SUBSTRING(string FROM integer) -{% endhighlight %} - -

    Returns a substring of a character string starting at a given point.

    -
    - {% highlight text %} -SUBSTRING(string FROM integer FOR integer) -{% endhighlight %} - -

    Returns a substring of a character string starting at a given point with a given length.

    -
    - {% highlight text %} -INITCAP(string) -{% endhighlight %} - -

    Returns string with the first letter of each word converter to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters.

    -
    - {% highlight text %} -CONCAT(string1, string2,...) -{% endhighlight %} - -

    Returns the string that results from concatenating the arguments. Returns NULL if any argument is NULL. E.g. CONCAT("AA", "BB", "CC") returns AABBCC.

    -
    - {% highlight text %} -CONCAT_WS(separator, string1, string2,...) -{% endhighlight %} - -

    Returns the string that results from concatenating the arguments using a separator. The separator is added between the strings to be concatenated. Returns NULL If the separator is NULL. CONCAT_WS() does not skip empty strings. However, it does skip any NULL argument. E.g. CONCAT_WS("~", "AA", "BB", "", "CC") returns AA~BB~~CC

    -
    - {% highlight text %} -LPAD(text string, len integer, pad string) -{% endhighlight %} - -

    Returns the string text left-padded with the string pad to a length of len characters. If text is longer than len, the return value is shortened to len characters. E.g. LPAD('hi',4,'??') returns ??hi, LPAD('hi',1,'??') returns h.

    -
    - {% highlight text %} -RPAD(text string, len integer, pad string) -{% endhighlight %} - -

    Returns the string text right-padded with the string pad to a length of len characters. If text is longer than len, the return value is shortened to len characters. E.g. RPAD('hi',4,'??') returns hi??, RPAD('hi',1,'??') returns h.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Conditional functionsDescription
    - {% highlight text %} -CASE value -WHEN value1 [, value11 ]* THEN result1 -[ WHEN valueN [, valueN1 ]* THEN resultN ]* -[ ELSE resultZ ] -END -{% endhighlight %} - -

    Simple case.

    -
    - {% highlight text %} -CASE -WHEN condition1 THEN result1 -[ WHEN conditionN THEN resultN ]* -[ ELSE resultZ ] -END -{% endhighlight %} - -

    Searched case.

    -
    - {% highlight text %} -NULLIF(value, value) -{% endhighlight %} - -

    Returns NULL if the values are the same. For example, NULLIF(5, 5) returns NULL; NULLIF(5, 0) returns 5.

    -
    - {% highlight text %} -COALESCE(value, value [, value ]* ) -{% endhighlight %} - -

    Provides a value if the first value is null. For example, COALESCE(NULL, 5) returns 5.

    -
    - - - - - - - - - - - - - - - -
    Type conversion functionsDescription
    - {% highlight text %} -CAST(value AS type) -{% endhighlight %} - -

    Converts a value to a given type.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Temporal functionsDescription
    - {% highlight text %} -DATE string -{% endhighlight %} - -

    Parses a date string in the form "yy-mm-dd" to a SQL date.

    -
    - {% highlight text %} -TIME string -{% endhighlight %} - -

    Parses a time string in the form "hh:mm:ss" to a SQL time.

    -
    - {% highlight text %} -TIMESTAMP string -{% endhighlight %} - -

    Parses a timestamp string in the form "yy-mm-dd hh:mm:ss.fff" to a SQL timestamp.

    -
    - {% highlight text %} -INTERVAL string range -{% endhighlight %} - -

    Parses an interval string in the form "dd hh:mm:ss.fff" for SQL intervals of milliseconds or "yyyy-mm" for SQL intervals of months. An interval range might be e.g. DAY, MINUTE, DAY TO HOUR, or DAY TO SECOND for intervals of milliseconds; YEAR or YEAR TO MONTH for intervals of months. E.g. INTERVAL '10 00:00:00.004' DAY TO SECOND, INTERVAL '10' DAY, or INTERVAL '2-10' YEAR TO MONTH return intervals.

    -
    - {% highlight text %} -CURRENT_DATE -{% endhighlight %} - -

    Returns the current SQL date in UTC time zone.

    -
    - {% highlight text %} -CURRENT_TIME -{% endhighlight %} - -

    Returns the current SQL time in UTC time zone.

    -
    - {% highlight text %} -CURRENT_TIMESTAMP -{% endhighlight %} - -

    Returns the current SQL timestamp in UTC time zone.

    -
    - {% highlight text %} -LOCALTIME -{% endhighlight %} - -

    Returns the current SQL time in local time zone.

    -
    - {% highlight text %} -LOCALTIMESTAMP -{% endhighlight %} - -

    Returns the current SQL timestamp in local time zone.

    -
    - {% highlight text %} -EXTRACT(timeintervalunit FROM temporal) -{% endhighlight %} - -

    Extracts parts of a time point or time interval. Returns the part as a long value. E.g. EXTRACT(DAY FROM DATE '2006-06-05') leads to 5.

    -
    - {% highlight text %} -FLOOR(timepoint TO timeintervalunit) -{% endhighlight %} - -

    Rounds a time point down to the given unit. E.g. FLOOR(TIME '12:44:31' TO MINUTE) leads to 12:44:00.

    -
    - {% highlight text %} -CEIL(timepoint TO timeintervalunit) -{% endhighlight %} - -

    Rounds a time point up to the given unit. E.g. CEIL(TIME '12:44:31' TO MINUTE) leads to 12:45:00.

    -
    - {% highlight text %} -QUARTER(date) -{% endhighlight %} - -

    Returns the quarter of a year from a SQL date. E.g. QUARTER(DATE '1994-09-27') leads to 3.

    -
    - {% highlight text %} -(timepoint, temporal) OVERLAPS (timepoint, temporal) -{% endhighlight %} - -

    Determines whether two anchored time intervals overlap. Time point and temporal are transformed into a range defined by two time points (start, end). The function evaluates leftEnd >= rightStart && rightEnd >= leftStart. E.g. (TIME '2:55:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:30:00', INTERVAL '2' HOUR) leads to true; (TIME '9:00:00', TIME '10:00:00') OVERLAPS (TIME '10:15:00', INTERVAL '3' HOUR) leads to false.

    -
    - {% highlight text %} -DATE_FORMAT(timestamp, format) -{% endhighlight %} - -

    Formats timestamp as a string using a specified format string. The format must be compatible with MySQL's date formatting syntax as used by the date_parse function. The format specification is given in the Date Format Specifier table below.

    -

    For example DATE_FORMAT(ts, '%Y, %d %M') results in strings formatted as "2017, 05 May".

    -
    - {% highlight text %} -TIMESTAMPADD(unit, interval, timestamp) -{% endhighlight %} - -

    Adds a (signed) integer interval to a timestamp. The unit for the interval is given by the unit argument, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR. E.g. TIMESTAMPADD(WEEK, 1, '2003-01-02') leads to 2003-01-09.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Aggregate functionsDescription
    - {% highlight text %} -COUNT(value [, value]* ) -{% endhighlight %} - -

    Returns the number of input rows for which value is not null.

    -
    - {% highlight text %} -COUNT(*) -{% endhighlight %} - -

    Returns the number of input rows.

    -
    - {% highlight text %} -AVG(numeric) -{% endhighlight %} - -

    Returns the average (arithmetic mean) of numeric across all input values.

    -
    - {% highlight text %} -SUM(numeric) -{% endhighlight %} - -

    Returns the sum of numeric across all input values.

    -
    - {% highlight text %} -MAX(value) -{% endhighlight %} - -

    Returns the maximum value of value across all input values.

    -
    - {% highlight text %} -MIN(value) -{% endhighlight %} - -

    Returns the minimum value of value across all input values.

    -
    - {% highlight text %} -STDDEV_POP(value) -{% endhighlight %} - -

    Returns the population standard deviation of the numeric field across all input values.

    -
    - {% highlight text %} -STDDEV_SAMP(value) -{% endhighlight %} - -

    Returns the sample standard deviation of the numeric field across all input values.

    -
    - {% highlight text %} -VAR_POP(value) -{% endhighlight %} - -

    Returns the population variance (square of the population standard deviation) of the numeric field across all input values.

    -
    - {% highlight text %} -VAR_SAMP(value) -{% endhighlight %} - -

    Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.

    -
    - {% highlight text %} - COLLECT(value) - {% endhighlight %} - -

    Returns a multiset of the values. null input value will be ignored. Return an empty multiset if only null values are added.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - -
    Grouping functionsDescription
    - {% highlight text %} -GROUP_ID() -{% endhighlight %} - -

    Returns an integer that uniquely identifies the combination of grouping keys.

    -
    - {% highlight text %} -GROUPING(expression) -{% endhighlight %} - -

    Returns 1 if expression is rolled up in the current row’s grouping set, 0 otherwise.

    -
    - {% highlight text %} -GROUPING_ID(expression [, expression]* ) -{% endhighlight %} - -

    Returns a bit vector of the given grouping expressions.

    -
    - - - - - - - - - - - - - - - - - - - - -
    Value access functionsDescription
    - {% highlight text %} -tableName.compositeType.field -{% endhighlight %} - -

    Accesses the field of a Flink composite type (such as Tuple, POJO, etc.) by name and returns it's value.

    -
    - {% highlight text %} -tableName.compositeType.* -{% endhighlight %} - -

    Converts a Flink composite type (such as Tuple, POJO, etc.) and all of its direct subtypes into a flat representation where every subtype is a separate field.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Value constructor functionsDescription
    - {% highlight text %} -(value, [, value]*) -{% endhighlight %} - -

    Creates a row from a list of values.

    -
    - {% highlight text %} -ROW(value, [, value]*) -{% endhighlight %} - -

    Creates a row from a list of values.

    -
    - {% highlight text %} -ARRAY ‘[’ value [, value ]* ‘]’ -{% endhighlight %} - -

    Creates an array from a list of values.

    -
    - {% highlight text %} -MAP ‘[’ key, value [, key, value ]* ‘]’ -{% endhighlight %} - -

    Creates a map from a list of key-value pairs.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Array functionsDescription
    - {% highlight text %} -CARDINALITY(ARRAY) -{% endhighlight %} - -

    Returns the number of elements of an array.

    -
    - {% highlight text %} -array ‘[’ index ‘]’ -{% endhighlight %} - -

    Returns the element at a particular position in an array. The index starts at 1.

    -
    - {% highlight text %} -ELEMENT(ARRAY) -{% endhighlight %} - -

    Returns the sole element of an array with a single element. Returns null if the array is empty. Throws an exception if the array has more than one element.

    -
    - - - - - - - - - - - - - - - - - - - - - -
    Map functionsDescription
    - {% highlight text %} -CARDINALITY(MAP) -{% endhighlight %} - -

    Returns the number of entries of a map.

    -
    - {% highlight text %} -map ‘[’ key ‘]’ -{% endhighlight %} - -

    Returns the value specified by a particular key in a map.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - -
    Hash functionsDescription
    - {% highlight text %} -MD5(string) -{% endhighlight %} - -

    Returns the MD5 hash of the string argument as a string of 32 hexadecimal digits; null if string is null.

    -
    - {% highlight text %} -SHA1(string) -{% endhighlight %} - -

    Returns the SHA-1 hash of the string argument as a string of 40 hexadecimal digits; null if string is null.

    -
    - {% highlight text %} -SHA256(string) -{% endhighlight %} - -

    Returns the SHA-256 hash of the string argument as a string of 64 hexadecimal digits; null if string is null.

    -
    - -### Unsupported Functions - -The following functions are not supported yet: - -- Binary string operators and functions -- System functions -- Distinct aggregate functions like COUNT DISTINCT - -{% top %} - -Reserved Keywords ------------------ - -Although not every SQL feature is implemented yet, some string combinations are already reserved as keywords for future use. If you want to use one of the following strings as a field name, make sure to surround them with backticks (e.g. `` `value` ``, `` `count` ``). - -{% highlight sql %} - -A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICTS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE - -{% endhighlight %} - -#### Date Format Specifier - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    SpecifierDescription
    {% highlight text %}%a{% endhighlight %}Abbreviated weekday name (Sun .. Sat)
    {% highlight text %}%b{% endhighlight %}Abbreviated month name (Jan .. Dec)
    {% highlight text %}%c{% endhighlight %}Month, numeric (1 .. 12)
    {% highlight text %}%D{% endhighlight %}Day of the month with English suffix (0th, 1st, 2nd, 3rd, ...)
    {% highlight text %}%d{% endhighlight %}Day of the month, numeric (01 .. 31)
    {% highlight text %}%e{% endhighlight %}Day of the month, numeric (1 .. 31)
    {% highlight text %}%f{% endhighlight %}Fraction of second (6 digits for printing: 000000 .. 999000; 1 - 9 digits for parsing: 0 .. 999999999) (Timestamp is truncated to milliseconds.)
    {% highlight text %}%H{% endhighlight %}Hour (00 .. 23)
    {% highlight text %}%h{% endhighlight %}Hour (01 .. 12)
    {% highlight text %}%I{% endhighlight %}Hour (01 .. 12)
    {% highlight text %}%i{% endhighlight %}Minutes, numeric (00 .. 59)
    {% highlight text %}%j{% endhighlight %}Day of year (001 .. 366)
    {% highlight text %}%k{% endhighlight %}Hour (0 .. 23)
    {% highlight text %}%l{% endhighlight %}Hour (1 .. 12)
    {% highlight text %}%M{% endhighlight %}Month name (January .. December)
    {% highlight text %}%m{% endhighlight %}Month, numeric (01 .. 12)
    {% highlight text %}%p{% endhighlight %}AM or PM
    {% highlight text %}%r{% endhighlight %}Time, 12-hour (hh:mm:ss followed by AM or PM)
    {% highlight text %}%S{% endhighlight %}Seconds (00 .. 59)
    {% highlight text %}%s{% endhighlight %}Seconds (00 .. 59)
    {% highlight text %}%T{% endhighlight %}Time, 24-hour (hh:mm:ss)
    {% highlight text %}%U{% endhighlight %}Week (00 .. 53), where Sunday is the first day of the week
    {% highlight text %}%u{% endhighlight %}Week (00 .. 53), where Monday is the first day of the week
    {% highlight text %}%V{% endhighlight %}Week (01 .. 53), where Sunday is the first day of the week; used with %X
    {% highlight text %}%v{% endhighlight %}Week (01 .. 53), where Monday is the first day of the week; used with %x
    {% highlight text %}%W{% endhighlight %}Weekday name (Sunday .. Saturday)
    {% highlight text %}%W{% endhighlight %}Weekday name (Sunday .. Saturday)
    {% highlight text %}%w{% endhighlight %}Day of the week (0 .. 6), where Sunday is the first day of the week
    {% highlight text %}%X{% endhighlight %}Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V
    {% highlight text %}%x{% endhighlight %}Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v
    {% highlight text %}%Y{% endhighlight %}Year, numeric, four digits
    {% highlight text %}%y{% endhighlight %}Year, numeric (two digits)
    {% highlight text %}%%{% endhighlight %}A literal % character
    {% highlight text %}%x{% endhighlight %}x, for any x not listed above
    {% top %} - diff --git a/docs/dev/table/sqlClient.md b/docs/dev/table/sqlClient.md new file mode 100644 index 00000000000000..97373f409f2fae --- /dev/null +++ b/docs/dev/table/sqlClient.md @@ -0,0 +1,558 @@ +--- +title: "SQL Client" +nav-parent_id: tableapi +nav-pos: 100 +is_beta: true +--- + + + +Flink’s Table & SQL API makes it possible to work with queries written in the SQL language, but these queries need to be embedded within a table program that is written in either Java or Scala. Moreover, these programs need to be packaged with a build tool before being submitted to a cluster. This more or less limits the usage of Flink to Java/Scala programmers. + +The *SQL Client* aims to provide an easy way of writing, debugging, and submitting table programs to a Flink cluster without a single line of Java or Scala code. The *SQL Client CLI* allows for retrieving and visualizing real-time results from the running distributed application on the command line. + +Animated demo of the Flink SQL Client CLI running table programs on a cluster + +Attention The SQL Client is in an early development phase. Even though the application is not production-ready yet, it can be a quite useful tool for prototyping and playing around with Flink SQL. In the future, the community plans to extend its functionality by providing a REST-based [SQL Client Gateway](sqlClient.html#limitations--future). + +* This will be replaced by the TOC +{:toc} + +Getting Started +--------------- + +This section describes how to setup and run your first Flink SQL program from the command-line. + +The SQL Client is bundled in the regular Flink distribution and thus runnable out-of-the-box. It requires only a running Flink cluster where table programs can be executed. For more information about setting up a Flink cluster see the [Cluster & Deployment]({{ site.baseurl }}/ops/deployment/cluster_setup.html) part. If you simply want to try out the SQL Client, you can also start a local cluster with one worker using the following command: + +{% highlight bash %} +./bin/start-cluster.sh +{% endhighlight %} + +### Starting the SQL Client CLI + +The SQL Client scripts are also located in the binary directory of Flink. [In the future](sqlClient.html#limitations--future), a user will have two possibilities of starting the SQL Client CLI either by starting an embedded standalone process or by connecting to a remote SQL Client Gateway. At the moment only the `embedded` mode is supported. You can start the CLI by calling: + +{% highlight bash %} +./bin/sql-client.sh embedded +{% endhighlight %} + +By default, the SQL Client will read its configuration from the environment file located in `./conf/sql-client-defaults.yaml`. See the [configuration part](sqlClient.html#environment-files) for more information about the structure of environment files. + +### Running SQL Queries + +Once the CLI has been started, you can use the `HELP` command to list all available SQL statements. For validating your setup and cluster connection, you can enter your first SQL query and press the `Enter` key to execute it: + +{% highlight sql %} +SELECT 'Hello World'; +{% endhighlight %} + +This query requires no table source and produces a single row result. The CLI will retrieve results from the cluster and visualize them. You can close the result view by pressing the `Q` key. + +The CLI supports **two modes** for maintaining and visualizing results. + +The **table mode** materializes results in memory and visualizes them in a regular, paginated table representation. It can be enabled by executing the following command in the CLI: + +{% highlight text %} +SET execution.result-mode=table; +{% endhighlight %} + +The **changelog mode** does not materialize results and visualizes the result stream that is produced by a [continuous query](streaming/dynamic_tables.html#continuous-queries) consisting of insertions (`+`) and retractions (`-`). + +{% highlight text %} +SET execution.result-mode=changelog; +{% endhighlight %} + +You can use the following query to see both result modes in action: + +{% highlight sql %} +SELECT name, COUNT(*) AS cnt FROM (VALUES ('Bob'), ('Alice'), ('Greg'), ('Bob')) AS NameTable(name) GROUP BY name; +{% endhighlight %} + +This query performs a bounded word count example. + +In *changelog mode*, the visualized changelog should be similar to: + +{% highlight text %} ++ Bob, 1 ++ Alice, 1 ++ Greg, 1 +- Bob, 1 ++ Bob, 2 +{% endhighlight %} + +In *table mode*, the visualized result table is continuously updated until the table program ends with: + +{% highlight text %} +Bob, 2 +Alice, 1 +Greg, 1 +{% endhighlight %} + +Both result modes can be useful during the prototyping of SQL queries. In both modes, results are stored in the Java heap memory of the SQL Client. In order to keep the CLI interface responsive, the changelog mode only shows the latest 1000 changes. The table mode allows for navigating through bigger results that are only limited by the available main memory and the configured [maximum number of rows](sqlClient.html#configuration) (`max-table-result-rows`). + +Attention Queries that are executed in a batch environment, can only be retrieved using the `table` result mode. + +After a query is defined, it can be submitted to the cluster as a long-running, detached Flink job. For this, a target system that stores the results needs to be specified using the [INSERT INTO statement](sqlClient.html#detached-sql-queries). The [configuration section](sqlClient.html#configuration) explains how to declare table sources for reading data, how to declare table sinks for writing data, and how to configure other table program properties. + +{% top %} + +Configuration +------------- + +The SQL Client can be started with the following optional CLI commands. They are discussed in detail in the subsequent paragraphs. + +{% highlight text %} +./bin/sql-client.sh embedded --help + +Mode "embedded" submits Flink jobs from the local machine. + + Syntax: embedded [OPTIONS] + "embedded" mode options: + -d,--defaults The environment properties with which + every new session is initialized. + Properties might be overwritten by + session properties. + -e,--environment The environment properties to be + imported into the session. It might + overwrite default environment + properties. + -h,--help Show the help message with + descriptions of all options. + -j,--jar A JAR file to be imported into the + session. The file might contain + user-defined classes needed for the + execution of statements such as + functions, table sources, or sinks. + Can be used multiple times. + -l,--library A JAR file directory with which every + new session is initialized. The files + might contain user-defined classes + needed for the execution of + statements such as functions, table + sources, or sinks. Can be used + multiple times. + -s,--session The identifier for a session. + 'default' is the default identifier. +{% endhighlight %} + +{% top %} + +### Environment Files + +A SQL query needs a configuration environment in which it is executed. The so-called *environment files* define available table sources and sinks, external catalogs, user-defined functions, and other properties required for execution and deployment. + +Every environment file is a regular [YAML file](http://yaml.org/). An example of such a file is presented below. + +{% highlight yaml %} +# Define tables here such as sources, sinks, views, or temporal tables. + +tables: + - name: MyTableSource + type: source-table + update-mode: append + connector: + type: filesystem + path: "/path/to/something.csv" + format: + type: csv + fields: + - name: MyField1 + type: INT + - name: MyField2 + type: VARCHAR + line-delimiter: "\n" + comment-prefix: "#" + schema: + - name: MyField1 + type: INT + - name: MyField2 + type: VARCHAR + - name: MyCustomView + type: view + query: "SELECT MyField2 FROM MyTableSource" + +# Define user-defined functions here. + +functions: + - name: myUDF + from: class + class: foo.bar.AggregateUDF + constructor: + - 7.6 + - false + +# Execution properties allow for changing the behavior of a table program. + +execution: + type: streaming # required: execution mode either 'batch' or 'streaming' + result-mode: table # required: either 'table' or 'changelog' + max-table-result-rows: 1000000 # optional: maximum number of maintained rows in + # 'table' mode (1000000 by default, smaller 1 means unlimited) + time-characteristic: event-time # optional: 'processing-time' or 'event-time' (default) + parallelism: 1 # optional: Flink's parallelism (1 by default) + periodic-watermarks-interval: 200 # optional: interval for periodic watermarks (200 ms by default) + max-parallelism: 16 # optional: Flink's maximum parallelism (128 by default) + min-idle-state-retention: 0 # optional: table program's minimum idle state time + max-idle-state-retention: 0 # optional: table program's maximum idle state time + restart-strategy: # optional: restart strategy + type: fallback # "fallback" to global restart strategy by default + +# Deployment properties allow for describing the cluster to which table programs are submitted to. + +deployment: + response-timeout: 5000 +{% endhighlight %} + +This configuration: + +- defines an environment with a table source `MyTableSource` that reads from a CSV file, +- defines a view `MyCustomView` that declares a virtual table using a SQL query, +- defines a user-defined function `myUDF` that can be instantiated using the class name and two constructor parameters, +- specifies a parallelism of 1 for queries executed in this streaming environment, +- specifies an event-time characteristic, and +- runs queries in the `table` result mode. + +Depending on the use case, a configuration can be split into multiple files. Therefore, environment files can be created for general purposes (*defaults environment file* using `--defaults`) as well as on a per-session basis (*session environment file* using `--environment`). Every CLI session is initialized with the default properties followed by the session properties. For example, the defaults environment file could specify all table sources that should be available for querying in every session whereas the session environment file only declares a specific state retention time and parallelism. Both default and session environment files can be passed when starting the CLI application. If no default environment file has been specified, the SQL Client searches for `./conf/sql-client-defaults.yaml` in Flink's configuration directory. + +Attention Properties that have been set within a CLI session (e.g. using the `SET` command) have highest precedence: + +{% highlight text %} +CLI commands > session environment file > defaults environment file +{% endhighlight %} + +#### Restart Strategies + +Restart strategies control how Flink jobs are restarted in case of a failure. Similar to [global restart strategies]({{ site.baseurl }}/dev/restart_strategies.html) for a Flink cluster, a more fine-grained restart configuration can be declared in an environment file. + +The following strategies are supported: + +{% highlight yaml %} +execution: + # falls back to the global strategy defined in flink-conf.yaml + restart-strategy: + type: fallback + + # job fails directly and no restart is attempted + restart-strategy: + type: none + + # attempts a given number of times to restart the job + restart-strategy: + type: fixed-delay + attempts: 3 # retries before job is declared as failed (default: Integer.MAX_VALUE) + delay: 10000 # delay in ms between retries (default: 10 s) + + # attempts as long as the maximum number of failures per time interval is not exceeded + restart-strategy: + type: failure-rate + max-failures-per-interval: 1 # retries in interval until failing (default: 1) + failure-rate-interval: 60000 # measuring interval in ms for failure rate + delay: 10000 # delay in ms between retries (default: 10 s) +{% endhighlight %} + +{% top %} + +### Dependencies + +The SQL Client does not require to setup a Java project using Maven or SBT. Instead, you can pass the dependencies as regular JAR files that get submitted to the cluster. You can either specify each JAR file separately (using `--jar`) or define entire library directories (using `--library`). For connectors to external systems (such as Apache Kafka) and corresponding data formats (such as JSON), Flink provides **ready-to-use JAR bundles**. These JAR files can be downloaded for each release from the Maven central repository. + +The full list of offered SQL JARs and documentation about how to use them can be found on the [connection to external systems page](connect.html). + +The following example shows an environment file that defines a table source reading JSON data from Apache Kafka. + +{% highlight yaml %} +tables: + - name: TaxiRides + type: source-table + update-mode: append + connector: + property-version: 1 + type: kafka + version: "0.11" + topic: TaxiRides + startup-mode: earliest-offset + properties: + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + - key: group.id + value: testGroup + format: + property-version: 1 + type: json + schema: "ROW" + schema: + - name: rideId + type: LONG + - name: lon + type: FLOAT + - name: lat + type: FLOAT + - name: rowTime + type: TIMESTAMP + rowtime: + timestamps: + type: "from-field" + from: "rideTime" + watermarks: + type: "periodic-bounded" + delay: "60000" + - name: procTime + type: TIMESTAMP + proctime: true +{% endhighlight %} + +The resulting schema of the `TaxiRide` table contains most of the fields of the JSON schema. Furthermore, it adds a rowtime attribute `rowTime` and processing-time attribute `procTime`. + +Both `connector` and `format` allow to define a property version (which is currently version `1`) for future backwards compatibility. + +{% top %} + +### User-defined Functions + +The SQL Client allows users to create custom, user-defined functions to be used in SQL queries. Currently, these functions are restricted to be defined programmatically in Java/Scala classes. + +In order to provide a user-defined function, you need to first implement and compile a function class that extends `ScalarFunction`, `AggregateFunction` or `TableFunction` (see [User-defined Functions]({{ site.baseurl }}/dev/table/udfs.html)). One or more functions can then be packaged into a dependency JAR for the SQL Client. + +All functions must be declared in an environment file before being called. For each item in the list of `functions`, one must specify + +- a `name` under which the function is registered, +- the source of the function using `from` (restricted to be `class` for now), +- the `class` which indicates the fully qualified class name of the function and an optional list of `constructor` parameters for instantiation. + +{% highlight yaml %} +functions: + - name: ... # required: name of the function + from: class # required: source of the function (can only be "class" for now) + class: ... # required: fully qualified class name of the function + constructor: # optimal: constructor parameters of the function class + - ... # optimal: a literal parameter with implicit type + - class: ... # optimal: full class name of the parameter + constructor: # optimal: constructor parameters of the parameter's class + - type: ... # optimal: type of the literal parameter + value: ... # optimal: value of the literal parameter +{% endhighlight %} + +Make sure that the order and types of the specified parameters strictly match one of the constructors of your function class. + +#### Constructor Parameters + +Depending on the user-defined function, it might be necessary to parameterize the implementation before using it in SQL statements. + +As shown in the example before, when declaring a user-defined function, a class can be configured using constructor parameters in one of the following three ways: + +**A literal value with implicit type:** The SQL Client will automatically derive the type according to the literal value itself. Currently, only values of `BOOLEAN`, `INT`, `DOUBLE` and `VARCHAR` are supported here. +If the automatic derivation does not work as expected (e.g., you need a VARCHAR `false`), use explicit types instead. + +{% highlight yaml %} +- true # -> BOOLEAN (case sensitive) +- 42 # -> INT +- 1234.222 # -> DOUBLE +- foo # -> VARCHAR +{% endhighlight %} + +**A literal value with explicit type:** Explicitly declare the parameter with `type` and `value` properties for type-safety. + +{% highlight yaml %} +- type: DECIMAL + value: 11111111111111111 +{% endhighlight %} + +The table below illustrates the supported Java parameter types and the corresponding SQL type strings. + +| Java type | SQL type | +| :---------------------- | :---------------- | +| `java.math.BigDecimal` | `DECIMAL` | +| `java.lang.Boolean` | `BOOLEAN` | +| `java.lang.Byte` | `TINYINT` | +| `java.lang.Double` | `DOUBLE` | +| `java.lang.Float` | `REAL`, `FLOAT` | +| `java.lang.Integer` | `INTEGER`, `INT` | +| `java.lang.Long` | `BIGINT` | +| `java.lang.Short` | `SMALLINT` | +| `java.lang.String` | `VARCHAR` | + +More types (e.g., `TIMESTAMP` or `ARRAY`), primitive types, and `null` are not supported yet. + +**A (nested) class instance:** Besides literal values, you can also create (nested) class instances for constructor parameters by specifying the `class` and `constructor` properties. +This process can be recursively performed until all the constructor parameters are represented with literal values. + +{% highlight yaml %} +- class: foo.bar.paramClass + constructor: + - StarryName + - class: java.lang.Integer + constructor: + - class: java.lang.String + constructor: + - type: VARCHAR + value: 3 +{% endhighlight %} + +{% top %} + +Detached SQL Queries +-------------------- + +In order to define end-to-end SQL pipelines, SQL's `INSERT INTO` statement can be used for submitting long-running, detached queries to a Flink cluster. These queries produce their results into an external system instead of the SQL Client. This allows for dealing with higher parallelism and larger amounts of data. The CLI itself does not have any control over a detached query after submission. + +{% highlight sql %} +INSERT INTO MyTableSink SELECT * FROM MyTableSource +{% endhighlight %} + +The table sink `MyTableSink` has to be declared in the environment file. See the [connection page](connect.html) for more information about supported external systems and their configuration. An example for an Apache Kafka table sink is shown below. + +{% highlight yaml %} +tables: + - name: MyTableSink + type: sink-table + update-mode: append + connector: + property-version: 1 + type: kafka + version: "0.11" + topic: OutputTopic + properties: + - key: zookeeper.connect + value: localhost:2181 + - key: bootstrap.servers + value: localhost:9092 + - key: group.id + value: testGroup + format: + property-version: 1 + type: json + derive-schema: true + schema: + - name: rideId + type: LONG + - name: lon + type: FLOAT + - name: lat + type: FLOAT + - name: rideTime + type: TIMESTAMP +{% endhighlight %} + +The SQL Client makes sure that a statement is successfully submitted to the cluster. Once the query is submitted, the CLI will show information about the Flink job. + +{% highlight text %} +[INFO] Table update statement has been successfully submitted to the cluster: +Cluster ID: StandaloneClusterId +Job ID: 6f922fe5cba87406ff23ae4a7bb79044 +Web interface: http://localhost:8081 +{% endhighlight %} + +Attention The SQL Client does not track the status of the running Flink job after submission. The CLI process can be shutdown after the submission without affecting the detached query. Flink's [restart strategy]({{ site.baseurl }}/dev/restart_strategies.html) takes care of the fault-tolerance. A query can be cancelled using Flink's web interface, command-line, or REST API. + +{% top %} + +SQL Views +--------- + +Views allow to define virtual tables from SQL queries. The view definition is parsed and validated immediately. However, the actual execution happens when the view is accessed during the submission of a general `INSERT INTO` or `SELECT` statement. + +Views can either be defined in [environment files](sqlClient.html#environment-files) or within the CLI session. + +The following example shows how to define multiple views in a file. The views are registered in the order in which they are defined in the environment file. Reference chains such as _view A depends on view B depends on view C_ are supported. + +{% highlight yaml %} +tables: + - name: MyTableSource + # ... + - name: MyRestrictedView + type: view + query: "SELECT MyField2 FROM MyTableSource" + - name: MyComplexView + type: view + query: > + SELECT MyField2 + 42, CAST(MyField1 AS VARCHAR) + FROM MyTableSource + WHERE MyField2 > 200 +{% endhighlight %} + +Similar to table sources and sinks, views defined in a session environment file have highest precedence. + +Views can also be created within a CLI session using the `CREATE VIEW` statement: + +{% highlight text %} +CREATE VIEW MyNewView AS SELECT MyField2 FROM MyTableSource; +{% endhighlight %} + +Views created within a CLI session can also be removed again using the `DROP VIEW` statement: + +{% highlight text %} +DROP VIEW MyNewView; +{% endhighlight %} + +Attention The definition of views in the CLI is limited to the mentioned syntax above. Defining a schema for views or escaping whitespaces in table names will be supported in future versions. + +{% top %} + +Temporal Tables +--------------- + +A [temporal table](./streaming/temporal_tables.html) allows for a (parameterized) view on a changing history table that returns the content of a table at a specific point in time. This is especially useful for joining a table with the content of another table at a particular timestamp. More information can be found in the [temporal table joins](./streaming/joins.html#join-with-a-temporal-table) page. + +The following example shows how to define a temporal table `SourceTemporalTable`: + +{% highlight yaml %} +tables: + + # Define the table source (or view) that contains updates to a temporal table + - name: HistorySource + type: source-table + update-mode: append + connector: # ... + format: # ... + schema: + - name: integerField + type: INT + - name: stringField + type: VARCHAR + - name: rowtimeField + type: TIMESTAMP + rowtime: + timestamps: + type: from-field + from: rowtimeField + watermarks: + type: from-source + + # Define a temporal table over the changing history table with time attribute and primary key + - name: SourceTemporalTable + type: temporal-table + history-table: HistorySource + primary-key: integerField + time-attribute: rowtimeField # could also be a proctime field +{% endhighlight %} + +As shown in the example, definitions of table sources, views, and temporal tables can be mixed with each other. They are registered in the order in which they are defined in the environment file. For example, a temporal table can reference a view which can depend on another view or table source. + +{% top %} + +Limitations & Future +-------------------- + +The current SQL Client implementation is in a very early development stage and might change in the future as part of the bigger Flink Improvement Proposal 24 ([FLIP-24](https://cwiki.apache.org/confluence/display/FLINK/FLIP-24+-+SQL+Client)). Feel free to join the discussion and open issue about bugs and features that you find useful. + +{% top %} diff --git a/docs/dev/table/streaming.md b/docs/dev/table/streaming.md deleted file mode 100644 index 310121e754f14b..00000000000000 --- a/docs/dev/table/streaming.md +++ /dev/null @@ -1,607 +0,0 @@ ---- -title: "Streaming Concepts" -nav-parent_id: tableapi -nav-pos: 10 ---- - - -Flink's [Table API](tableApi.html) and [SQL support](sql.html) are unified APIs for batch and stream processing. This means that Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. Because the relational algebra and SQL were originally designed for batch processing, relational queries on unbounded streaming input are not as well understood as relational queries on bounded batch input. - -On this page, we explain concepts, practical limitations, and stream-specific configuration parameters of Flink's relational APIs on streaming data. - -* This will be replaced by the TOC -{:toc} - -Relational Queries on Data Streams ----------------------------------- - -SQL and the relational algebra have not been designed with streaming data in mind. As a consequence, there are few conceptual gaps between relational algebra (and SQL) and stream processing. - - - - - - - - - - - - - - - - - - -
    Relational Algebra / SQLStream Processing
    Relations (or tables) are bounded (multi-)sets of tuples.A stream is an infinite sequences of tuples.
    A query that is executed on batch data (e.g., a table in a relational database) has access to the complete input data.A streaming query cannot access all data when is started and has to "wait" for data to be streamed in.
    A batch query terminates after it produced a fixed sized result.A streaming query continuously updates its result based on the received records and never completes.
    - -Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update materialized views and updates a materialized view as soon as its base tables are updated. - -The connection between eager view maintenance and SQL queries on streams becomes obvious if we consider the following: - -- A database table is the result of a *stream* of `INSERT`, `UPDATE`, and `DELETE` DML statements, often called *changelog stream*. -- A materialized view is defined as a SQL query. In order to update the view, the query is continuously processes the changelog streams of the view's base relations. -- The materialized view is the result of the streaming SQL query. - -With these points in mind, we introduce Flink's concept of *Dynamic Tables* in the next section. - -Dynamic Tables & Continuous Queries ---------------------------------------- - -*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic table are changing over time. They can be queried like static batch tables. Querying a dynamic table yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its input (dynamic) table. Essentially, a continuous query on a dynamic table is very similar to the definition query of a materialized view. - -It is important to note that the result of a continuous query is always semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables. - -The following figure visualizes the relationship of streams, dynamic tables, and continuous queries: - -
    -Dynamic tables -
    - -1. A stream is converted into a dynamic table. -1. A continuous query is evaluated on the dynamic table yielding a new dynamic table. -1. The resulting dynamic table is converted back into a stream. - -**Note:** Dynamic tables are foremost a logical concept. Dynamic tables are not necessarily (fully) materialized during query execution. - -In the following, we will explain the concepts of dynamic tables and continuous queries with a stream of click events that have the following schema: - -``` -[ - user: VARCHAR, // the name of the user - cTime: TIMESTAMP, // the time when the URL was accessed - url: VARCHAR // the URL that was accessed by the user -] -``` - -### Defining a Table on a Stream - -In order to process a stream with a relational query, it has to be converted into a `Table`. Conceptually, each record of the stream is interpreted as an `INSERT` modification on the resulting table. Essentially, we are building a table from an `INSERT`-only changelog stream. - -The following figure visualizes how the stream of click event (left-hand side) is converted into a table (right-hand side). The resulting table is continuously growing as more records of the click stream are inserted. - -
    -Append mode -
    - -**Note:** A table which is defined on a stream is internally not materialized. - -### Continuous Queries - -A continuous query is evaluated on a dynamic table and produces a new dynamic table as result. In contrast to a batch query, a continuous query never terminates and updates its result table according to the updates on its input tables. At any point in time, the result of a continuous query is semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables. - -In the following we show two example queries on a `clicks` table that is defined on the stream of click events. - -The first query is a simple `GROUP-BY COUNT` aggregation query. It groups the `clicks` table on the `user` field and counts the number of visited URLs. The following figure shows how the query is evaluated over time as the `clicks` table is updated with additional rows. - -
    -Continuous Non-Windowed Query -
    - -When the query is started, the `clicks` table (left-hand side) is empty. The query starts to compute the result table, when the first row is inserted into the `clicks` table. After the first row `[Mary, ./home]` was inserted, the result table (right-hand side, top) consists of a single row `[Mary, 1]`. When the second row `[Bob, ./cart]` is inserted into the `clicks` table, the query updates the result table and inserts a new row `[Bob, 1]`. The third row `[Mary, ./prod?id=1]` yields an update of an already computed result row such that `[Mary, 1]` is updated to `[Mary, 2]`. Finally, the query inserts a third row `[Liz, 1]` into the result table, when the fourth row is appended to the `clicks` table. - -The second query is similar to the first one but groups the `clicks` table in addition to the `user` attribute also on an [hourly tumbling window](./sql.html#group-windows) before it counts the number of URLs (time-based computations such as windows are based on special [time attributes](#time-attributes), which are discussed below.). Again, the figure shows the input and output at different points in time to visualize the changing nature of dynamic tables. - -
    -Continuous Group-Window Query -
    - -As before, the input table `clicks` is shown on the left. The query continuously computes results every hour and updates the result table. The clicks table contains four rows with timestamps (`cTime`) between `12:00:00` and `12:59:59`. The query computes two results rows from this input (one for each `user`) and appends them to the result table. For the next window between `13:00:00` and `13:59:59`, the `clicks` table contains three rows, which results in another two rows being appended to the result table. The result table is updated, as more rows are appended to `clicks` over time. - -#### Update and Append Queries - -Although the two example queries appear to be quite similar (both compute a grouped count aggregate), they differ in one important aspect: -- The first query updates previously emitted results, i.e., the changelog stream that defines the result table contains `INSERT` and `UPDATE` changes. -- The second query only appends to the result table, i.e., the changelog stream of the result table only consists of `INSERT` changes. - -Whether a query produces an append-only table or an updated table has some implications: -- Queries that produce update changes usually have to maintain more state (see the following section). -- The conversion of an append-only table into a stream is different from the conversion of an updated table (see the [Table to Stream Conversion](#table-to-stream-conversion) section). - -#### Query Restrictions - -Many, but not all, semantically valid queries can be evaluated as continuous queries on streams. Some queries are too expensive to compute, either due to the size of state that they need to maintain or because computing updates is too expensive. - -- **State Size:** Continuous queries are evaluated on unbounded streams and are often supposed to run for weeks or months. Hence, the total amount of data that a continuous query processes can be very large. Queries that have to update previously emitted results need to maintain all emitted rows in order to be able to update them. For instance, the first example query needs to store the URL count for each user to be able to increase the count and sent out a new result when the input table receives a new row. If only registered users are tracked, the number of counts to maintain might not be too high. However, if non-registered users get a unique user name assigned, the number of counts to maintain would grow over time and might eventually cause the query to fail. - -{% highlight sql %} -SELECT user, COUNT(url) -FROM clicks -GROUP BY user; -{% endhighlight %} - -- **Computing Updates:** Some queries require to recompute and update a large fraction of the emitted result rows even if only a single input record is added or updated. Clearly, such queries are not well suited to be executed as continuous queries. An example is the following query which computes for each user a `RANK` based on the time of the last click. As soon as the `clicks` table receives a new row, the `lastAction` of the user is updated and a new rank must be computed. However since two rows cannot have the same rank, all lower ranked rows need to be updated as well. - -{% highlight sql %} -SELECT user, RANK() OVER (ORDER BY lastLogin) -FROM ( - SELECT user, MAX(cTime) AS lastAction FROM clicks GROUP BY user -); -{% endhighlight %} - -The [QueryConfig](#query-configuration) section discusses parameters to control the execution of continuous queries. Some parameters can be used to trade the size of maintained state for result accuracy. - -### Table to Stream Conversion - -A dynamic table can be continuously modified by `INSERT`, `UPDATE`, and `DELETE` changes just like a regular database table. It might be a table with a single row, which is constantly updated, an insert-only table without `UPDATE` and `DELETE` modifications, or anything in between. - -When converting a dynamic table into a stream or writing it to an external system, these changes need to be encoded. Flink's Table API and SQL support three ways to encode the changes of a dynamic table: - -* **Append-only stream:** A dynamic table that is only modified by `INSERT` changes can be converted into a stream by emitting the inserted rows. - -* **Retract stream:** A retract stream is a stream with two types of messages, *add messages* and *retract messages*. A dynamic table is converted into an retract stream by encoding an `INSERT` change as add message, a `DELETE` change as retract message, and an `UPDATE` change as a retract message for the updated (previous) row and an add message for the updating (new) row. The following figure visualizes the conversion of a dynamic table into a retract stream. - -
    -Dynamic tables -
    -

    - -* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete message*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a dynamic table by encoding `INSERT` and `UPDATE` changes as upsert message and `DELETE` changes as delete message. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream. - -
    -Dynamic tables -
    -

    - -The API to convert a dynamic table into a `DataStream` is discussed on the [Common Concepts](./common.html#convert-a-table-into-a-datastream) page. Please note that only append and retract streams are supported when converting a dynamic table into a `DataStream`. The `TableSink` interface to emit a dynamic table to an external system are discussed on the [TableSources and TableSinks](./sourceSinks.html#define-a-tablesink) page. - -{% top %} - -Time Attributes ---------------- - -Flink is able to process streaming data based on different notions of *time*. - -- *Processing time* refers to the system time of the machine (also known as "wall-clock time") that is executing the respective operation. -- *Event time* refers to the processing of streaming data based on timestamps which are attached to each row. The timestamps can encode when an event happened. -- *Ingestion time* is the time that events enter Flink; internally, it is treated similarly to event time. - -For more information about time handling in Flink, see the introduction about [Event Time and Watermarks]({{ site.baseurl }}/dev/event_time.html). - -Table programs require that the corresponding time characteristic has been specified for the streaming environment: - -
    -
    -{% highlight java %} -final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - -env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); // default - -// alternatively: -// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); -// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); -{% endhighlight %} -
    -
    -{% highlight scala %} -val env = StreamExecutionEnvironment.getExecutionEnvironment - -env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) // default - -// alternatively: -// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) -// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) -{% endhighlight %} -
    -
    - -Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs. - -Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can used in time-based operations. - -As long as a time attribute is not modified and is simply forwarded from one part of the query to another, it remains a valid time attribute. Time attributes behave like regular timestamps and can be accessed for calculations. If a time attribute is used in a calculation, it will be materialized and becomes a regular timestamp. Regular timestamps do not cooperate with Flink's time and watermarking system and thus can not be used for time-based operations anymore. - -### Processing time - -Processing time allows a table program to produce results based on the time of the local machine. It is the simplest notion of time but does not provide determinism. It neither requires timestamp extraction nor watermark generation. - -There are two ways to define a processing time attribute. - -#### During DataStream-to-Table Conversion - -The processing time attribute is defined with the `.proctime` property during schema definition. The time attribute must only extend the physical schema by an additional logical field. Thus, it can only be defined at the end of the schema definition. - -
    -
    -{% highlight java %} -DataStream> stream = ...; - -// declare an additional logical field as a processing time attribute -Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.proctime"); - -WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); -{% endhighlight %} -
    -
    -{% highlight scala %} -val stream: DataStream[(String, String)] = ... - -// declare an additional logical field as a processing time attribute -val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'Username, 'Data, 'UserActionTime.proctime) - -val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) -{% endhighlight %} -
    -
    - -#### Using a TableSource - -The processing time attribute is defined by a `TableSource` that implements the `DefinedProctimeAttribute` interface. The logical time attribute is appended to the physical schema defined by the return type of the `TableSource`. - -
    -
    -{% highlight java %} -// define a table source with a processing attribute -public class UserActionSource implements StreamTableSource, DefinedProctimeAttribute { - - @Override - public TypeInformation getReturnType() { - String[] names = new String[] {"Username" , "Data"}; - TypeInformation[] types = new TypeInformation[] {Types.STRING(), Types.STRING()}; - return Types.ROW(names, types); - } - - @Override - public DataStream getDataStream(StreamExecutionEnvironment execEnv) { - // create stream - DataStream stream = ...; - return stream; - } - - @Override - public String getProctimeAttribute() { - // field with this name will be appended as a third field - return "UserActionTime"; - } -} - -// register table source -tEnv.registerTableSource("UserActions", new UserActionSource()); - -WindowedTable windowedTable = tEnv - .scan("UserActions") - .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); -{% endhighlight %} -
    -
    -{% highlight scala %} -// define a table source with a processing attribute -class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttribute { - - override def getReturnType = { - val names = Array[String]("Username" , "Data") - val types = Array[TypeInformation[_]](Types.STRING, Types.STRING) - Types.ROW(names, types) - } - - override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = { - // create stream - val stream = ... - stream - } - - override def getProctimeAttribute = { - // field with this name will be appended as a third field - "UserActionTime" - } -} - -// register table source -tEnv.registerTableSource("UserActions", new UserActionSource) - -val windowedTable = tEnv - .scan("UserActions") - .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) -{% endhighlight %} -
    -
    - -### Event time - -Event time allows a table program to produce results based on the time that is contained in every record. This allows for consistent results even in case of out-of-order events or late events. It also ensures replayable results of the table program when reading records from persistent storage. - -Additionally, event time allows for unified syntax for table programs in both batch and streaming environments. A time attribute in a streaming environment can be a regular field of a record in a batch environment. - -In order to handle out-of-order events and distinguish between on-time and late events in streaming, Flink needs to extract timestamps from events and make some kind of progress in time (so-called [watermarks]({{ site.baseurl }}/dev/event_time.html)). - -An event time attribute can be defined either during DataStream-to-Table conversion or by using a TableSource. - -#### During DataStream-to-Table Conversion - -The event time attribute is defined with the `.rowtime` property during schema definition. [Timestamps and watermarks]({{ site.baseurl }}/dev/event_time.html) must have been assigned in the `DataStream` that is converted. - -There are two ways of defining the time attribute when converting a `DataStream` into a `Table`. Depending on whether the specified `.rowtime` field name exists in the schema of the `DataStream` or not, the timestamp field is either - -- appended as a new field to the schema or -- replaces an existing field. - -In either case the event time timestamp field will hold the value of the `DataStream` event time timestamp. - -
    -
    -{% highlight java %} - -// Option 1: - -// extract timestamp and assign watermarks based on knowledge of the stream -DataStream> stream = inputStream.assignTimestampsAndWatermarks(...); - -// declare an additional logical field as an event time attribute -Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.rowtime"); - - -// Option 2: - -// extract timestamp from first field, and assign watermarks based on knowledge of the stream -DataStream> stream = inputStream.assignTimestampsAndWatermarks(...); - -// the first field has been used for timestamp extraction, and is no longer necessary -// replace first field with a logical event time attribute -Table table = tEnv.fromDataStream(stream, "UserActionTime.rowtime, Username, Data"); - -// Usage: - -WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); -{% endhighlight %} -
    -
    -{% highlight scala %} - -// Option 1: - -// extract timestamp and assign watermarks based on knowledge of the stream -val stream: DataStream[(String, String)] = inputStream.assignTimestampsAndWatermarks(...) - -// declare an additional logical field as an event time attribute -val table = tEnv.fromDataStream(stream, 'Username, 'Data, 'UserActionTime.rowtime) - - -// Option 2: - -// extract timestamp from first field, and assign watermarks based on knowledge of the stream -val stream: DataStream[(Long, String, String)] = inputStream.assignTimestampsAndWatermarks(...) - -// the first field has been used for timestamp extraction, and is no longer necessary -// replace first field with a logical event time attribute -val table = tEnv.fromDataStream(stream, 'UserActionTime.rowtime, 'Username, 'Data) - -// Usage: - -val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) -{% endhighlight %} -
    -
    - -#### Using a TableSource - -The event time attribute is defined by a `TableSource` that implements the `DefinedRowtimeAttribute` interface. The `getRowtimeAttribute()` method returns the name of an existing field that carries the event time attribute of the table and is of type `LONG` or `TIMESTAMP`. - -Moreover, the `DataStream` returned by the `getDataStream()` method must have watermarks assigned that are aligned with the defined time attribute. Please note that the timestamps of the `DataStream` (the ones which are assigned by a `TimestampAssigner`) are ignored. Only the values of the `TableSource`'s rowtime attribute are relevant. - -
    -
    -{% highlight java %} -// define a table source with a rowtime attribute -public class UserActionSource implements StreamTableSource, DefinedRowtimeAttribute { - - @Override - public TypeInformation getReturnType() { - String[] names = new String[] {"Username", "Data", "UserActionTime"}; - TypeInformation[] types = - new TypeInformation[] {Types.STRING(), Types.STRING(), Types.LONG()}; - return Types.ROW(names, types); - } - - @Override - public DataStream getDataStream(StreamExecutionEnvironment execEnv) { - // create stream - // ... - // assign watermarks based on the "UserActionTime" attribute - DataStream stream = inputStream.assignTimestampsAndWatermarks(...); - return stream; - } - - @Override - public String getRowtimeAttribute() { - // Mark the "UserActionTime" attribute as event-time attribute. - return "UserActionTime"; - } -} - -// register the table source -tEnv.registerTableSource("UserActions", new UserActionSource()); - -WindowedTable windowedTable = tEnv - .scan("UserActions") - .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); -{% endhighlight %} -
    -
    -{% highlight scala %} -// define a table source with a rowtime attribute -class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribute { - - override def getReturnType = { - val names = Array[String]("Username" , "Data", "UserActionTime") - val types = Array[TypeInformation[_]](Types.STRING, Types.STRING, Types.LONG) - Types.ROW(names, types) - } - - override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = { - // create stream - // ... - // assign watermarks based on the "UserActionTime" attribute - val stream = inputStream.assignTimestampsAndWatermarks(...) - stream - } - - override def getRowtimeAttribute = { - // Mark the "UserActionTime" attribute as event-time attribute. - "UserActionTime" - } -} - -// register the table source -tEnv.registerTableSource("UserActions", new UserActionSource) - -val windowedTable = tEnv - .scan("UserActions") - .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) -{% endhighlight %} -
    -
    - -{% top %} - -Query Configuration -------------------- - -Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. In many cases, continuous queries on streaming input are capable of computing accurate results that are identical to offline computed results. However, this is not possible in general case because continuous queries have to restrict the size of the state they are maintaining in order to avoid to run out of storage and to be able to process unbounded streaming data over a long period of time. As a result, a continuous query might only be able to provide approximated results depending on the characteristics of the input data and the query itself. - -Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream](common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](common.html#emit-a-table). - -
    -
    -{% highlight java %} -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); -StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); - -// obtain query configuration from TableEnvironment -StreamQueryConfig qConfig = tableEnv.queryConfig(); -// set query parameters -qConfig.withIdleStateRetentionTime(Time.hours(12)); - -// define query -Table result = ... - -// create TableSink -TableSink sink = ... - -// emit result Table via a TableSink -result.writeToSink(sink, qConfig); - -// convert result Table into a DataStream -DataStream stream = tableEnv.toAppendStream(result, Row.class, qConfig); - -{% endhighlight %} -
    -
    -{% highlight scala %} -val env = StreamExecutionEnvironment.getExecutionEnvironment -val tableEnv = TableEnvironment.getTableEnvironment(env) - -// obtain query configuration from TableEnvironment -val qConfig: StreamQueryConfig = tableEnv.queryConfig -// set query parameters -qConfig.withIdleStateRetentionTime(Time.hours(12)) - -// define query -val result: Table = ??? - -// create TableSink -val sink: TableSink[Row] = ??? - -// emit result Table via a TableSink -result.writeToSink(sink, qConfig) - -// convert result Table into a DataStream[Row] -val stream: DataStream[Row] = result.toAppendStream[Row](qConfig) - -{% endhighlight %} -
    -
    - -In the following we describe the parameters of the `QueryConfig` and how they affect the accuracy and resource consumption of a query. - -### Idle State Retention Time - -Many queries aggregate or join records on one or more key attributes. When such a query is executed on a stream, the continuous query needs to collect records or maintain partial results per key. If the key domain of the input stream is evolving, i.e., the active key values are changing over time, the continuous query accumulates more and more state as more and more distinct keys are observed. However, often keys become inactive after some time and their corresponding state becomes stale and useless. - -For example the following query computes the number of clicks per session. - -``` -SELECT sessionId, COUNT(*) FROM clicks GROUP BY sessionId; -``` - -The `sessionId` attribute is used as a grouping key and the continuous query maintains a count for each `sessionId` it observes. The `sessionId` attribute is evolving over time and `sessionId` values are only active until the session ends, i.e., for a limited period of time. However, the continuous query cannot know about this property of `sessionId` and expects that every `sessionId` value can occur at any point of time. It maintains a count for each observed `sessionId` value. Consequently, the total state size of the query is continuously growing as more and more `sessionId` values are observed. - -The *Idle State Retention Time* parameters define for how long the state of a key is retained without being updated before it is removed. For the previous example query, the count of a `sessionId` would be removed as soon as it has not been updated for the configured period of time. - -By removing the state of a key, the continuous query completely forgets that it has seen this key before. If a record with a key, whose state has been removed before, is processed, the record will be treated as if it was the first record with the respective key. For the example above this means that the count of a `sessionId` would start again at `0`. - -There are two parameters to configure the idle state retention time: -- The *minimum idle state retention time* defines how long the state of an inactive key is at least kept before it is removed. -- The *maximum idle state retention time* defines how long the state of an inactive key is at most kept before it is removed. - -The parameters are specified as follows: - -
    -
    -{% highlight java %} - -StreamQueryConfig qConfig = ... - -// set idle state retention time: min = 12 hour, max = 16 hours -qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(16)); -// set idle state retention time. min = max = 12 hours -qConfig.withIdleStateRetentionTime(Time.hours(12); - -{% endhighlight %} -
    -
    -{% highlight scala %} - -val qConfig: StreamQueryConfig = ??? - -// set idle state retention time: min = 12 hour, max = 16 hours -qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(16)) -// set idle state retention time. min = max = 12 hours -qConfig.withIdleStateRetentionTime(Time.hours(12) - -{% endhighlight %} -
    -
    - -Configuring different minimum and maximum idle state retention times is more efficient because it reduces the internal book-keeping of a query for when to remove state. - -{% top %} - - diff --git a/docs/dev/table/streaming/dynamic_tables.md b/docs/dev/table/streaming/dynamic_tables.md new file mode 100644 index 00000000000000..f8bcb9405c61b7 --- /dev/null +++ b/docs/dev/table/streaming/dynamic_tables.md @@ -0,0 +1,189 @@ +--- +title: "Dynamic Tables" +nav-parent_id: streaming_tableapi +nav-pos: 1 +--- + + +SQL and the relational algebra have not been designed with streaming data in mind. As a consequence, there are few conceptual gaps between relational algebra (and SQL) and stream processing. + +This page discusses these differences and explains how Flink can achieve the same semantics on unbounded data as a regular database engine on bounded data. + +* This will be replaced by the TOC +{:toc} + +Relational Queries on Data Streams +---------------------------------- + +The following table compares traditional relational algebra and stream processing with respect to input data, execution, and output results. + + + + + + + + + + + + + + + + + + +
    Relational Algebra / SQLStream Processing
    Relations (or tables) are bounded (multi-)sets of tuples.A stream is an infinite sequences of tuples.
    A query that is executed on batch data (e.g., a table in a relational database) has access to the complete input data.A streaming query cannot access all data when is started and has to "wait" for data to be streamed in.
    A batch query terminates after it produced a fixed sized result.A streaming query continuously updates its result based on the received records and never completes.
    + +Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update a materialized view as soon as its base tables are updated. + +The connection between eager view maintenance and SQL queries on streams becomes obvious if we consider the following: + +- A database table is the result of a *stream* of `INSERT`, `UPDATE`, and `DELETE` DML statements, often called *changelog stream*. +- A materialized view is defined as a SQL query. In order to update the view, the query continuously processes the changelog streams of the view's base relations. +- The materialized view is the result of the streaming SQL query. + +With these points in mind, we introduce following concept of *Dynamic tables* in the next section. + +Dynamic Tables & Continuous Queries +--------------------------------------- + +*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic tables are changing over time. They can be queried like static batch tables. Querying dynamic tables yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its (dynamic) input tables. Essentially, a continuous query on a dynamic table is very similar to a query that defines a materialized view. + +It is important to note that the result of a continuous query is always semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables. + +The following figure visualizes the relationship of streams, dynamic tables, and continuous queries: + +
    +Dynamic tables +
    + +1. A stream is converted into a dynamic table. +1. A continuous query is evaluated on the dynamic table yielding a new dynamic table. +1. The resulting dynamic table is converted back into a stream. + +**Note:** Dynamic tables are foremost a logical concept. Dynamic tables are not necessarily (fully) materialized during query execution. + +In the following, we will explain the concepts of dynamic tables and continuous queries with a stream of click events that have the following schema: + +{% highlight plain %} +[ + user: VARCHAR, // the name of the user + cTime: TIMESTAMP, // the time when the URL was accessed + url: VARCHAR // the URL that was accessed by the user +] +{% endhighlight %} + +Defining a Table on a Stream +---------------------------- + +In order to process a stream with a relational query, it has to be converted into a `Table`. Conceptually, each record of the stream is interpreted as an `INSERT` modification on the resulting table. Essentially, we are building a table from an `INSERT`-only changelog stream. + +The following figure visualizes how the stream of click event (left-hand side) is converted into a table (right-hand side). The resulting table is continuously growing as more records of the click stream are inserted. + +
    +Append mode +
    + +**Note:** A table which is defined on a stream is internally not materialized. + +### Continuous Queries +---------------------- + +A continuous query is evaluated on a dynamic table and produces a new dynamic table as result. In contrast to a batch query, a continuous query never terminates and updates its result table according to the updates on its input tables. At any point in time, the result of a continuous query is semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables. + +In the following we show two example queries on a `clicks` table that is defined on the stream of click events. + +The first query is a simple `GROUP-BY COUNT` aggregation query. It groups the `clicks` table on the `user` field and counts the number of visited URLs. The following figure shows how the query is evaluated over time as the `clicks` table is updated with additional rows. + +
    +Continuous Non-Windowed Query +
    + +When the query is started, the `clicks` table (left-hand side) is empty. The query starts to compute the result table, when the first row is inserted into the `clicks` table. After the first row `[Mary, ./home]` was inserted, the result table (right-hand side, top) consists of a single row `[Mary, 1]`. When the second row `[Bob, ./cart]` is inserted into the `clicks` table, the query updates the result table and inserts a new row `[Bob, 1]`. The third row `[Mary, ./prod?id=1]` yields an update of an already computed result row such that `[Mary, 1]` is updated to `[Mary, 2]`. Finally, the query inserts a third row `[Liz, 1]` into the result table, when the fourth row is appended to the `clicks` table. + +The second query is similar to the first one but groups the `clicks` table in addition to the `user` attribute also on an [hourly tumbling window](../sql.html#group-windows) before it counts the number of URLs (time-based computations such as windows are based on special [time attributes](time_attributes.html), which are discussed later.). Again, the figure shows the input and output at different points in time to visualize the changing nature of dynamic tables. + +
    +Continuous Group-Window Query +
    + +As before, the input table `clicks` is shown on the left. The query continuously computes results every hour and updates the result table. The clicks table contains four rows with timestamps (`cTime`) between `12:00:00` and `12:59:59`. The query computes two results rows from this input (one for each `user`) and appends them to the result table. For the next window between `13:00:00` and `13:59:59`, the `clicks` table contains three rows, which results in another two rows being appended to the result table. The result table is updated, as more rows are appended to `clicks` over time. + +### Update and Append Queries + +Although the two example queries appear to be quite similar (both compute a grouped count aggregate), they differ in one important aspect: +- The first query updates previously emitted results, i.e., the changelog stream that defines the result table contains `INSERT` and `UPDATE` changes. +- The second query only appends to the result table, i.e., the changelog stream of the result table only consists of `INSERT` changes. + +Whether a query produces an append-only table or an updated table has some implications: +- Queries that produce update changes usually have to maintain more state (see the following section). +- The conversion of an append-only table into a stream is different from the conversion of an updated table (see the [Table to Stream Conversion](#table-to-stream-conversion) section). + +### Query Restrictions + +Many, but not all, semantically valid queries can be evaluated as continuous queries on streams. Some queries are too expensive to compute, either due to the size of state that they need to maintain or because computing updates is too expensive. + +- **State Size:** Continuous queries are evaluated on unbounded streams and are often supposed to run for weeks or months. Hence, the total amount of data that a continuous query processes can be very large. Queries that have to update previously emitted results need to maintain all emitted rows in order to be able to update them. For instance, the first example query needs to store the URL count for each user to be able to increase the count and sent out a new result when the input table receives a new row. If only registered users are tracked, the number of counts to maintain might not be too high. However, if non-registered users get a unique user name assigned, the number of counts to maintain would grow over time and might eventually cause the query to fail. + +{% highlight sql %} +SELECT user, COUNT(url) +FROM clicks +GROUP BY user; +{% endhighlight %} + +- **Computing Updates:** Some queries require to recompute and update a large fraction of the emitted result rows even if only a single input record is added or updated. Clearly, such queries are not well suited to be executed as continuous queries. An example is the following query which computes for each user a `RANK` based on the time of the last click. As soon as the `clicks` table receives a new row, the `lastAction` of the user is updated and a new rank must be computed. However since two rows cannot have the same rank, all lower ranked rows need to be updated as well. + +{% highlight sql %} +SELECT user, RANK() OVER (ORDER BY lastLogin) +FROM ( + SELECT user, MAX(cTime) AS lastAction FROM clicks GROUP BY user +); +{% endhighlight %} + +The [Query Configuration](query_configuration.html) page discusses parameters to control the execution of continuous queries. Some parameters can be used to trade the size of maintained state for result accuracy. + +Table to Stream Conversion +-------------------------- + +A dynamic table can be continuously modified by `INSERT`, `UPDATE`, and `DELETE` changes just like a regular database table. It might be a table with a single row, which is constantly updated, an insert-only table without `UPDATE` and `DELETE` modifications, or anything in between. + +When converting a dynamic table into a stream or writing it to an external system, these changes need to be encoded. Flink's Table API and SQL support three ways to encode the changes of a dynamic table: + +* **Append-only stream:** A dynamic table that is only modified by `INSERT` changes can be converted into a stream by emitting the inserted rows. + +* **Retract stream:** A retract stream is a stream with two types of messages, *add messages* and *retract messages*. A dynamic table is converted into an retract stream by encoding an `INSERT` change as add message, a `DELETE` change as retract message, and an `UPDATE` change as a retract message for the updated (previous) row and an add message for the updating (new) row. The following figure visualizes the conversion of a dynamic table into a retract stream. + +
    +Dynamic tables +
    +

    + +* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete messages*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a stream by encoding `INSERT` and `UPDATE` changes as upsert messages and `DELETE` changes as delete messages. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream. + +
    +Dynamic tables +
    +

    + +The API to convert a dynamic table into a `DataStream` is discussed on the [Common Concepts](../common.html#convert-a-table-into-a-datastream) page. Please note that only append and retract streams are supported when converting a dynamic table into a `DataStream`. The `TableSink` interface to emit a dynamic table to an external system are discussed on the [TableSources and TableSinks](../sourceSinks.html#define-a-tablesink) page. + +{% top %} diff --git a/docs/dev/table/streaming/index.md b/docs/dev/table/streaming/index.md new file mode 100644 index 00000000000000..a6fa525a3dbbc8 --- /dev/null +++ b/docs/dev/table/streaming/index.md @@ -0,0 +1,44 @@ +--- +title: "Streaming Concepts" +nav-id: streaming_tableapi +nav-parent_id: tableapi +nav-pos: 10 +is_beta: false +nav-show_overview: true +--- + + +Flink's [Table API](../tableApi.html) and [SQL support](../sql.html) are unified APIs for batch and stream processing. +This means that Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. +Because the relational algebra and SQL were originally designed for batch processing, +relational queries on unbounded streaming input are not as well understood as relational queries on bounded batch input. + +The following pages explain concepts, practical limitations, and stream-specific configuration parameters of Flink's relational APIs on streaming data. + +Where to go next? +----------------- + +* [Dynamic Tables]({{ site.baseurl }}/dev/table/streaming/dynamic_tables.html): Describes the concept of dynamic tables. +* [Time attributes]({{ site.baseurl }}/dev/table/streaming/time_attributes.html): Explains time attributes and how time attributes are handled in Table API & SQL. +* [Joins in Continuous Queries]({{ site.baseurl }}/dev/table/streaming/joins.html): Different supported types of Joins in Continuous Queries. +* [Temporal Tables]({{ site.baseurl }}/dev/table/streaming/temporal_tables.html): Describes the Temporal Table concept. +* [Query configuration]({{ site.baseurl }}/dev/table/streaming/query_configuration.html): Lists Table API & SQL specific configuration options. + +{% top %} diff --git a/docs/dev/table/streaming/joins.md b/docs/dev/table/streaming/joins.md new file mode 100644 index 00000000000000..a572c1be293cf4 --- /dev/null +++ b/docs/dev/table/streaming/joins.md @@ -0,0 +1,225 @@ +--- +title: "Joins in Continuous Queries" +nav-parent_id: streaming_tableapi +nav-pos: 3 +--- + + +Joins are a common and well-understood operation in batch data processing to connect the rows of two relations. However, the semantics of joins on [dynamic tables](dynamic_tables.html) are much less obvious or even confusing. + +Because of that, there are a couple of ways to actually perform a join using either Table API or SQL. + +For more information regarding the syntax, please check the join sections in [Table API](../tableApi.html#joins) and [SQL](../sql.html#joins). + +* This will be replaced by the TOC +{:toc} + +Regular Joins +------------- + +Regular joins are the most generic type of join in which any new records or changes to either side of the join input are visible and are affecting the whole join result. +For example, if there is a new record on the left side, it will be joined with all of the previous and future records on the right side. + +{% highlight sql %} +SELECT * FROM Orders +INNER JOIN Product +ON Orders.productId = Product.id +{% endhighlight %} + +These semantics allow for any kind of updating (insert, update, delete) input tables. + +However, this operation has an important implication: it requires to keep both sides of the join input in Flink's state forever. +Thus, the resource usage will grow indefinitely as well, if one or both input tables are continuously growing. + +Time-windowed Joins +------------------- + +A time-windowed join is defined by a join predicate, that checks if the [time attributes](time_attributes.html) of the input +records are within certain time constraints, i.e., a time window. + +{% highlight sql %} +SELECT * +FROM + Orders o, + Shipments s +WHERE o.id = s.orderId AND + o.ordertime BETWEEN s.shiptime - INTERVAL '4' HOUR AND s.shiptime +{% endhighlight %} + +Compared to a regular join operation, this kind of join only supports append-only tables with time attributes. Since time attributes are quasi-monontic increasing, Flink can remove old values from its state without affecting the correctness of the result. + +Join with a Temporal Table +-------------------------- + +A join with a temporal table joins an append-only table (left input/probe side) with a temporal table (right input/build side), +i.e., a table that changes over time and tracks its changes. Please check the corresponding page for more information about [temporal tables](temporal_tables.html). + +The following example shows an append-only table `Orders` that should be joined with the continuously changing currency rates table `RatesHistory`. + +`Orders` is an append-only table that represents payments for the given `amount` and the given `currency`. +For example at `10:15` there was an order for an amount of `2 Euro`. + +{% highlight sql %} +SELECT * FROM Orders; + +rowtime amount currency +======= ====== ========= +10:15 2 Euro +10:30 1 US Dollar +10:32 50 Yen +10:52 3 Euro +11:04 5 US Dollar +{% endhighlight %} + +`RatesHistory` represents an ever changing append-only table of currency exchange rates with respect to `Yen` (which has a rate of `1`). +For example, the exchange rate for the period from `09:00` to `10:45` of `Euro` to `Yen` was `114`. From `10:45` to `11:15` it was `116`. + +{% highlight sql %} +SELECT * FROM RatesHistory; + +rowtime currency rate +======= ======== ====== +09:00 US Dollar 102 +09:00 Euro 114 +09:00 Yen 1 +10:45 Euro 116 +11:15 Euro 119 +11:49 Pounds 108 +{% endhighlight %} + +Given that we would like to calculate the amount of all `Orders` converted to a common currency (`Yen`). + +For example, we would like to convert the following order using the appropriate conversion rate for the given `rowtime` (`114`). + +{% highlight text %} +rowtime amount currency +======= ====== ========= +10:15 2 Euro +{% endhighlight %} + +Without using the concept of [temporal tables](temporal_tables.html), one would need to write a query like: + +{% highlight sql %} +SELECT + SUM(o.amount * r.rate) AS amount +FROM Orders AS o, + RatesHistory AS r +WHERE r.currency = o.currency +AND r.rowtime = ( + SELECT MAX(rowtime) + FROM RatesHistory AS r2 + WHERE r2.currency = o.currency + AND r2.rowtime <= o.rowtime); +{% endhighlight %} + +With the help of a temporal table function `Rates` over `RatesHistory`, we can express such a query in SQL as: + +{% highlight sql %} +SELECT + o.amount * r.rate AS amount +FROM + Orders AS o, + LATERAL TABLE (Rates(o.rowtime)) AS r +WHERE r.currency = o.currency +{% endhighlight %} + +Each record from the probe side will be joined with the version of the build side table at the time of the correlated time attribute of the probe side record. +In order to support updates (overwrites) of previous values on the build side table, the table must define a primary key. + +In our example, each record from `Orders` will be joined with the version of `Rates` at time `o.rowtime`. The `currency` field has been defined as the primary key of `Rates` before and is used to connect both tables in our example. If the query were using a processing-time notion, a newly appended order would always be joined with the most recent version of `Rates` when executing the operation. + +In contrast to [regular joins](#regular-joins), this means that if there is a new record on the build side, it will not affect the previous results of the join. +This again allows Flink to limit the number of elements that must be kept in the state. + +Compared to [time-windowed joins](#time-windowed-joins), temporal table joins do not define a time window within which bounds the records will be joined. +Records from the probe side are always joined with the build side's version at the time specified by the time attribute. Thus, records on the build side might be arbitrarily old. +As time passes, the previous and no longer needed versions of the record (for the given primary key) will be removed from the state. + +Such behaviour makes a temporal table join a good candidate to express stream enrichment in relational terms. + +### Usage + +After [defining temporal table function](temporal_tables.html#defining-temporal-table-function), we can start using it. +Temporal table functions can be used in the same way as normal table functions would be used. + +The following code snippet solves our motivating problem of converting currencies from the `Orders` table: + +
    +
    +{% highlight sql %} +SELECT + SUM(o_amount * r_rate) AS amount +FROM + Orders, + LATERAL TABLE (Rates(o_proctime)) +WHERE + r_currency = o_currency +{% endhighlight %} +
    +
    +{% highlight java %} +Table result = orders + .join(new Table(tEnv, "rates(o_proctime)"), "o_currency = r_currency") + .select("(o_amount * r_rate).sum as amount"); +{% endhighlight %} +
    +
    +{% highlight scala %} +val result = orders + .join(rates('o_proctime), 'r_currency === 'o_currency) + .select(('o_amount * 'r_rate).sum as 'amount) +{% endhighlight %} +
    +
    + +**Note**: State retention defined in a [query configuration](query_configuration.html) is not yet implemented for temporal joins. +This means that the required state to compute the query result might grow infinitely depending on the number of distinct primary keys for the history table. + +### Processing-time Temporal Joins + +With a processing-time time attribute, it is impossible to pass _past_ time attributes as an argument to the temporal table function. +By definition, it is always the current timestamp. Thus, invocations of a processing-time temporal table function will always return the latest known versions of the underlying table +and any updates in the underlying history table will also immediately overwrite the current values. + +Only the latest versions (with respect to the defined primary key) of the build side records are kept in the state. +Updates of the build side will have no effect on previously emitted join results. + +One can think about a processing-time temporal join as a simple `HashMap` that stores all of the records from the build side. +When a new record from the build side has the same key as some previous record, the old value is just simply overwritten. +Every record from the probe side is always evaluated against the most recent/current state of the `HashMap`. + +### Event-time Temporal Joins + +With an event-time time attribute (i.e., a rowtime attribute), it is possible to pass _past_ time attributes to the temporal table function. +This allows for joining the two tables at a common point in time. + +Compared to processing-time temporal joins, the temporal table does not only keep the latest version (with respect to the defined primary key) of the build side records in the state +but stores all versions (identified by time) since the last watermark. + +For example, an incoming row with an event-time timestamp of `12:30:00` that is appended to the probe side table +is joined with the version of the build side table at time `12:30:00` according to the [concept of temporal tables](temporal_tables.html). +Thus, the incoming row is only joined with rows that have a timestamp lower or equal to `12:30:00` with +applied updates according to the primary key until this point in time. + +By definition of event time, [watermarks]({{ site.baseurl }}/dev/event_time.html) allow the join operation to move +forward in time and discard versions of the build table that are no longer necessary because no incoming row with +lower or equal timestamp is expected. + +{% top %} diff --git a/docs/dev/table/streaming/match_recognize.md b/docs/dev/table/streaming/match_recognize.md new file mode 100644 index 00000000000000..4090440f8645c0 --- /dev/null +++ b/docs/dev/table/streaming/match_recognize.md @@ -0,0 +1,964 @@ +--- +title: 'Detecting Patterns in Tables' +nav-parent_id: streaming_tableapi +nav-title: 'Detecting Patterns' +nav-pos: 5 +is_beta: true +--- + + +It is a common use case to search for a set of event patterns, especially in case of data streams. Flink +comes with a [complex event processing (CEP) library]({{ site.baseurl }}/dev/libs/cep.html) which allows for pattern detection in event streams. Furthermore, Flink's +SQL API provides a relational way of expressing queries with a large set of built-in functions and rule-based optimizations that can be used out of the box. + +In December 2016, the International Organization for Standardization (ISO) released a new version of the SQL standard which includes _Row Pattern Recognition in SQL_ ([ISO/IEC TR 19075-5:2016](https://standards.iso.org/ittf/PubliclyAvailableStandards/c065143_ISO_IEC_TR_19075-5_2016.zip)). It allows Flink to consolidate CEP and SQL API using the `MATCH_RECOGNIZE` clause for complex event processing in SQL. + +A `MATCH_RECOGNIZE` clause enables the following tasks: +* Logically partition and order the data that is used with the `PARTITION BY` and `ORDER BY` clauses. +* Define patterns of rows to seek using the `PATTERN` clause. These patterns use a syntax similar to that of regular expressions. +* The logical components of the row pattern variables are specified in the `DEFINE` clause. +* Define measures, which are expressions usable in other parts of the SQL query, in the `MEASURES` clause. + +The following example illustrates the syntax for basic pattern recognition: + +{% highlight sql %} +SELECT T.aid, T.bid, T.cid +FROM MyTable + MATCH_RECOGNIZE ( + PARTITION BY userid + ORDER BY proctime + MEASURES + A.id AS aid, + B.id AS bid, + C.id AS cid + PATTERN (A B C) + DEFINE + A AS name = 'a', + B AS name = 'b', + C AS name = 'c' + ) AS T +{% endhighlight %} + +This page will explain each keyword in more detail and will illustrate more complex examples. + +Attention Flink's implementation of the `MATCH_RECOGNIZE` clause is a subset of the full standard. Only those features documented in the following sections are supported. Since the development is still in an early phase, please also take a look at the [known limitations](#known-limitations). + +* This will be replaced by the TOC +{:toc} + +Introduction and Examples +------------------------- + +### Installation Guide + +The pattern recognition feature uses the Apache Flink's CEP library internally. In order to be able to use the `MATCH_RECOGNIZE` clause, +the library needs to be added as a dependency to your Maven project. + +{% highlight xml %} + + org.apache.flink + flink-cep{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + +Alternatively, you can also add the dependency to the cluster classpath (see the [dependency section]({{ site.baseurl}}/projectsetup/dependencies.html) for more information). + +If you want to use the `MATCH_RECOGNIZE` clause in the [SQL Client]({{ site.baseurl}}/dev/table/sqlClient.html), +you don't have to do anything as all the dependencies are included by default. + +### SQL Semantics + +Every `MATCH_RECOGNIZE` query consists of the following clauses: + +* [PARTITION BY](#partitioning) - defines the logical partitioning of the table; similar to a `GROUP BY` operation. +* [ORDER BY](#order-of-events) - specifies how the incoming rows should be ordered; this is essential as patterns depend on an order. +* [MEASURES](#define--measures) - defines output of the clause; similar to a `SELECT` clause. +* [ONE ROW PER MATCH](#output-mode) - output mode which defines how many rows per match should be produced. +* [AFTER MATCH SKIP](#after-match-strategy) - specifies where the next match should start; this is also a way to control how many distinct matches a single event can belong to. +* [PATTERN](#defining-a-pattern) - allows constructing patterns that will be searched for using a _regular expression_-like syntax. +* [DEFINE](#define--measures) - this section defines the conditions that the pattern variables must satisfy. + +Attention Currently, the `MATCH_RECOGNIZE` clause can only be applied to an [append table](dynamic_tables.html#update-and-append-queries). Furthermore, it always produces +an append table as well. + +### Examples + +For our examples, we assume that a table `Ticker` has been registered. The table contains prices of stocks at a particular point in time. + +The table has a following schema: + +{% highlight text %} +Ticker + |-- symbol: String # symbol of the stock + |-- price: Long # price of the stock + |-- tax: Long # tax liability of the stock + |-- rowtime: TimeIndicatorTypeInfo(rowtime) # point in time when the change to those values happened +{% endhighlight %} + +For simplification, we only consider the incoming data for a single stock `ACME`. A ticker could look similar to the following table where rows are continuously appended. + +{% highlight text %} +symbol rowtime price tax +====== ==================== ======= ======= +'ACME' '01-Apr-11 10:00:00' 12 1 +'ACME' '01-Apr-11 10:00:01' 17 2 +'ACME' '01-Apr-11 10:00:02' 19 1 +'ACME' '01-Apr-11 10:00:03' 21 3 +'ACME' '01-Apr-11 10:00:04' 25 2 +'ACME' '01-Apr-11 10:00:05' 18 1 +'ACME' '01-Apr-11 10:00:06' 15 1 +'ACME' '01-Apr-11 10:00:07' 14 2 +'ACME' '01-Apr-11 10:00:08' 24 2 +'ACME' '01-Apr-11 10:00:09' 25 2 +'ACME' '01-Apr-11 10:00:10' 19 1 +{% endhighlight %} + +The task is now to find periods of a constantly decreasing price of a single ticker. For this, one could write a query like: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE ( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + START_ROW.rowtime AS start_tstamp, + LAST(PRICE_DOWN.rowtime) AS bottom_tstamp, + LAST(PRICE_UP.rowtime) AS end_tstamp + ONE ROW PER MATCH + AFTER MATCH SKIP TO LAST PRICE_UP + PATTERN (START_ROW PRICE_DOWN+ PRICE_UP) + DEFINE + PRICE_DOWN AS + (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR + PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1), + PRICE_UP AS + PRICE_UP.price > LAST(PRICE_DOWN.price, 1) + ) MR; +{% endhighlight %} + +The query partitions the `Ticker` table by the `symbol` column and orders it by the `rowtime` time attribute. + +The `PATTERN` clause specifies that we are interested in a pattern with a starting event `START_ROW` that is followed by one or more `PRICE_DOWN` events and concluded with a `PRICE_UP` event. If such a pattern can be found, the next pattern match will be seeked at the last `PRICE_UP` event as indicated by the `AFTER MATCH SKIP TO LAST` clause. + +The `DEFINE` clause specifies the conditions that need to be met for a `PRICE_DOWN` and `PRICE_UP` event. Although the `START_ROW` pattern variable is not present it has an implicit condition that is evaluated always as `TRUE`. + +A pattern variable `PRICE_DOWN` is defined as a row with a price that is smaller than the price of the last row that met the `PRICE_DOWN` condition. For the initial case or when there is no last row that met the `PRICE_DOWN` condition, the price of the row should be smaller than the price of the preceding row in the pattern (referenced by `START_ROW`). + +A pattern variable `PRICE_UP` is defined as a row with a price that is larger than the price of the last row that met the `PRICE_DOWN` condition. + +This query produces a summary row for each period in which the price of a stock was continuously decreasing. + +The exact representation of the output rows is defined in the `MEASURES` part of the query. The number of output rows is defined by the `ONE ROW PER MATCH` output mode. + +{% highlight text %} + symbol start_tstamp bottom_tstamp end_tstamp +========= ================== ================== ================== +ACME 01-APR-11 10:00:04 01-APR-11 10:00:07 01-APR-11 10:00:08 +{% endhighlight %} + +The resulting row describes a period of falling prices that started at `01-APR-11 10:00:04` and +achieved the lowest price at `01-APR-11 10:00:07` that increased again at `01-APR-11 10:00:08`. + +Partitioning +------------ + +It is possible to look for patterns in partitioned data, e.g., trends for a single ticker or a particular user. This can be expressed using the `PARTITION BY` clause. The clause is similar to using `GROUP BY` for aggregations. + +Attention It is highly advised to partition the incoming data because otherwise the `MATCH_RECOGNIZE` clause will be translated +into a non-parallel operator to ensure global ordering. + +Order of Events +--------------- + +Apache Flink allows for searching for patterns based on time; either [processing time or event time](time_attributes.html). + +In case of event time, the events are sorted before they are passed to the internal pattern state machine. As a consequence, the +produced output will be correct regardless of the order in which rows are appended to the table. Instead, the pattern is evaluated in the order specified by the time contained in each row. + +The `MATCH_RECOGNIZE` clause assumes a [time attribute](time_attributes.html) with ascending ordering as the first argument to `ORDER BY` clause. + +For the example `Ticker` table, a definition like `ORDER BY rowtime ASC, price DESC` is valid but `ORDER BY price, rowtime` or `ORDER BY rowtime DESC, price ASC` is not. + +Define & Measures +----------------- + +The `DEFINE` and `MEASURES` keywords have similar meanings to the `WHERE` and `SELECT` clauses in a simple SQL query. + +The `MEASURES` clause defines what will be included in the output of a matching pattern. It can project columns and define expressions for evaluation. +The number of produced rows depends on the [output mode](#output-mode) setting. + +The `DEFINE` clause specifies conditions that rows have to fulfill in order to be classified to a corresponding [pattern variable](#defining-a-pattern). +If a condition is not defined for a pattern variable, a default condition will be used which evaluates to `true` for every row. + +For a more detailed explanation about expressions that can be used in those clauses, please have a look at the [event stream navigation](#pattern-navigation) section. + +### Aggregations + +Aggregations can be used in `DEFINE` and `MEASURES` clauses. Both [built-in]({{ site.baseurl }}/dev/table/sql.html#built-in-functions) and custom [user defined]({{ site.baseurl }}/dev/table/udfs.html) functions are supported. + +Aggregate functions are applied to each subset of rows mapped to a match. In order to understand how those subsets are evaluated have a look at the [event stream navigation](#pattern-navigation) section. + +The task of the following example is to find the longest period of time for which the average price of a ticker did not go below certain threshold. It shows how expressible `MATCH_RECOGNIZE` can become with aggregations. +This task can be performed with the following query: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE ( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + FIRST(A.rowtime) AS start_tstamp, + LAST(A.rowtime) AS end_tstamp, + AVG(A.price) AS avgPrice + ONE ROW PER MATCH + AFTER MATCH SKIP TO FIRST B + PATTERN (A+ B) + DEFINE + A AS AVG(A.price) < 15 + ) MR; +{% endhighlight %} + +Given this query and following input values: + +{% highlight text %} +symbol rowtime price tax +====== ==================== ======= ======= +'ACME' '01-Apr-11 10:00:00' 12 1 +'ACME' '01-Apr-11 10:00:01' 17 2 +'ACME' '01-Apr-11 10:00:02' 13 1 +'ACME' '01-Apr-11 10:00:03' 16 3 +'ACME' '01-Apr-11 10:00:04' 25 2 +'ACME' '01-Apr-11 10:00:05' 2 1 +'ACME' '01-Apr-11 10:00:06' 4 1 +'ACME' '01-Apr-11 10:00:07' 10 2 +'ACME' '01-Apr-11 10:00:08' 15 2 +'ACME' '01-Apr-11 10:00:09' 25 2 +'ACME' '01-Apr-11 10:00:10' 30 1 +{% endhighlight %} + +The query will accumulate events as part of the pattern variable `A` as long as the average price of them does not exceed `15`. For example, such a limit exceeding happens at `01-Apr-11 10:00:04`. +The following period exceeds the average price of `15` again at `01-Apr-11 10:00:10`. Thus the results for said query will be: + +{% highlight text %} + symbol start_tstamp end_tstamp avgPrice +========= ================== ================== ============ +ACME 01-APR-11 10:00:00 01-APR-11 10:00:03 14.5 +ACME 01-APR-11 10:00:04 01-APR-11 10:00:09 13.5 +{% endhighlight %} + +Note Aggregations can be applied to expressions, but only if they reference a single pattern variable. Thus `SUM(A.price * A.tax)` is a valid one, but `AVG(A.price * B.tax)` is not. + +Attention `DISTINCT` aggregations are not supported. + +Defining a Pattern +------------------ + +The `MATCH_RECOGNIZE` clause allows users to search for patterns in event streams using a powerful and expressive syntax +that is somewhat similar to the widespread regular expression syntax. + +Every pattern is constructed from basic building blocks, called _pattern variables_, to which operators (quantifiers and other modifiers) can be applied. The whole pattern must be enclosed in +brackets. + +An example pattern could look like: + +{% highlight sql %} +PATTERN (A B+ C* D) +{% endhighlight %} + +One may use the following operators: + +* _Concatenation_ - a pattern like `(A B)` means that the contiguity is strict between `A` and `B`. Therefore, there can be no rows that were not mapped to `A` or `B` in between. +* _Quantifiers_ - modify the number of rows that can be mapped to the pattern variable. + * `*` — _0_ or more rows + * `+` — _1_ or more rows + * `?` — _0_ or _1_ rows + * `{ n }` — exactly _n_ rows (_n > 0_) + * `{ n, }` — _n_ or more rows (_n ≥ 0_) + * `{ n, m }` — between _n_ and _m_ (inclusive) rows (_0 ≤ n ≤ m, 0 < m_) + * `{ , m }` — between _0_ and _m_ (inclusive) rows (_m > 0_) + +Attention Patterns that can potentially produce an empty match are not supported. +Examples of such patterns are `PATTERN (A*)`, `PATTERN (A? B*)`, `PATTERN (A{0,} B{0,} C*)`, etc. + +### Greedy & Reluctant Quantifiers + +Each quantifier can be either _greedy_ (default behavior) or _reluctant_. Greedy quantifiers try to match +as many rows as possible while reluctant quantifiers try to match as few as possible. + +In order to illustrate the difference, one can view the following example with a query where a greedy quantifier is applied to the `B` variable: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + C.price AS lastPrice + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A B* C) + DEFINE + A AS A.price > 10, + B AS B.price < 15, + C AS C.price > 12 + ) +{% endhighlight %} + +Given we have the following input: + +{% highlight text %} + symbol tax price rowtime +======= ===== ======== ===================== + XYZ 1 10 2018-09-17 10:00:02 + XYZ 2 11 2018-09-17 10:00:03 + XYZ 1 12 2018-09-17 10:00:04 + XYZ 2 13 2018-09-17 10:00:05 + XYZ 1 14 2018-09-17 10:00:06 + XYZ 2 16 2018-09-17 10:00:07 +{% endhighlight %} + +The pattern above will produce the following output: + +{% highlight text %} + symbol lastPrice +======== =========== + XYZ 16 +{% endhighlight %} + +The same query where `B*` is modified to `B*?`, which means that `B*` should be reluctant, will produce: + +{% highlight text %} + symbol lastPrice +======== =========== + XYZ 13 + XYZ 16 +{% endhighlight %} + +The pattern variable `B` matches only to the row with price `12` instead of swallowing the rows with prices `12`, `13`, and `14`. + +Attention It is not possible to use a greedy quantifier for the last +variable of a pattern. Thus, a pattern like `(A B*)` is not allowed. This can be easily worked around by introducing an artificial state +(e.g. `C`) that has a negated condition of `B`. So you could use a query like: + +{% highlight sql %} +PATTERN (A B* C) +DEFINE + A AS condA(), + B AS condB(), + C AS NOT condB() +{% endhighlight %} + +Attention The optional reluctant quantifier (`A??` or `A{0,1}?`) is not supported right now. + +### Time constraint + +Especially for streaming use cases, it is often required that a pattern finishes within a given period of time. +This allows for limiting the overall state size that Flink has to maintain internally, even in case of greedy quantifiers. + +Therefore, Flink SQL supports the additional (non-standard SQL) `WITHIN` clause for defining a time constraint for a pattern. The clause can be defined after the `PATTERN` clause and takes an interval of millisecond resolution. + +If the time between the first and last event of a potential match is longer than the given value, such a match will not be appended to the result table. + +Note It is generally encouraged to use the `WITHIN` clause as it helps Flink with efficient memory management. Underlying state can be pruned once the threshold is reached. + +Attention However, the `WITHIN` clause is not part of the SQL standard. The recommended way of dealing with time constraints might change in the future. + +The use of the `WITHIN` clause is illustrated in the following example query: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + C.rowtime AS dropTime, + A.price - C.price AS dropDiff + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A B* C) WITHIN INTERVAL '1' HOUR + DEFINE + B AS B.price > A.price - 10 + C AS C.price < A.price - 10 + ) +{% endhighlight %} + +The query detects a price drop of `10` that happens within an interval of 1 hour. + +Let's assume the query is used to analyze the following ticker data: + +{% highlight text %} +symbol rowtime price tax +====== ==================== ======= ======= +'ACME' '01-Apr-11 10:00:00' 20 1 +'ACME' '01-Apr-11 10:20:00' 17 2 +'ACME' '01-Apr-11 10:40:00' 18 1 +'ACME' '01-Apr-11 11:00:00' 11 3 +'ACME' '01-Apr-11 11:20:00' 14 2 +'ACME' '01-Apr-11 11:40:00' 9 1 +'ACME' '01-Apr-11 12:00:00' 15 1 +'ACME' '01-Apr-11 12:20:00' 14 2 +'ACME' '01-Apr-11 12:40:00' 24 2 +'ACME' '01-Apr-11 13:00:00' 1 2 +'ACME' '01-Apr-11 13:20:00' 19 1 +{% endhighlight %} + +The query will produce the following results: + +{% highlight text %} +symbol dropTime dropDiff +====== ==================== ============= +'ACME' '01-Apr-11 13:00:00' 14 +{% endhighlight %} + +The resulting row represents a price drop from `15` (at `01-Apr-11 12:00:00`) to `1` (at `01-Apr-11 13:00:00`). The `dropDiff` column contains the price difference. + +Notice that even though prices also drop by higher values, for example, by `11` (between `01-Apr-11 10:00:00` and `01-Apr-11 11:40:00`), the time difference between those two events is larger than 1 hour. Thus, they don't produce a match. + +Output Mode +----------- + +The _output mode_ describes how many rows should be emitted for every found match. The SQL standard describes two modes: +- `ALL ROWS PER MATCH` +- `ONE ROW PER MATCH`. + +Currently, the only supported output mode is `ONE ROW PER MATCH` that will always produce one output summary row for each found match. + +The schema of the output row will be a concatenation of `[partitioning columns] + [measures columns]` in that particular order. + +The following example shows the output of a query defined as: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + FIRST(A.price) AS startPrice, + LAST(A.price) AS topPrice, + B.price AS lastPrice + ONE ROW PER MATCH + PATTERN (A+ B) + DEFINE + A AS LAST(A.price, 1) IS NULL OR A.price > LAST(A.price, 1), + B AS B.price < LAST(A.price) + ) +{% endhighlight %} + +For the following input rows: + +{% highlight text %} + symbol tax price rowtime +======== ===== ======== ===================== + XYZ 1 10 2018-09-17 10:00:02 + XYZ 2 12 2018-09-17 10:00:03 + XYZ 1 13 2018-09-17 10:00:04 + XYZ 2 11 2018-09-17 10:00:05 +{% endhighlight %} + +The query will produce the following output: + +{% highlight text %} + symbol startPrice topPrice lastPrice +======== ============ ========== =========== + XYZ 10 13 11 +{% endhighlight %} + +The pattern recognition is partitioned by the `symbol` column. Even though not explicitly mentioned in the `MEASURES` clause, the partitioned column is added at the beginning of the result. + +Pattern Navigation +------------------ + +The `DEFINE` and `MEASURES` clauses allow for navigating within the list of rows that (potentially) match a pattern. + +This section discusses this navigation for declaring conditions or producing output results. + +### Pattern Variable Referencing + +A _pattern variable reference_ allows a set of rows mapped to a particular pattern variable in the `DEFINE` or `MEASURES` clauses to be referenced. + +For example, the expression `A.price` describes a set of rows mapped so far to `A` plus the current row +if we try to match the current row to `A`. If an expression in the `DEFINE`/`MEASURES` clause requires a single row (e.g. `A.price` or `A.price > 10`), +it selects the last value belonging to the corresponding set. + +If no pattern variable is specified (e.g. `SUM(price)`), an expression references the default pattern variable `*` which references all variables in the pattern. +In other words, it creates a list of all the rows mapped so far to any variable plus the current row. + +#### Example + +For a more thorough example, one can take a look at the following pattern and corresponding conditions: + +{% highlight sql %} +PATTERN (A B+) +DEFINE + A AS A.price > 10, + B AS B.price > A.price AND SUM(price) < 100 AND SUM(B.price) < 80 +{% endhighlight %} + +The following table describes how those conditions are evaluated for each incoming event. + +The table consists of the following columns: + * `#` - the row identifier that uniquely identifies an incoming row in the lists `[A.price]`/`[B.price]`/`[price]`. + * `price` - the price of the incoming row. + * `[A.price]`/`[B.price]`/`[price]` - describe lists of rows which are used in the `DEFINE` clause to evaluate conditions. + * `Classifier` - the classifier of the current row which indicates the pattern variable the row is mapped to. + * `A.price`/`B.price`/`SUM(price)`/`SUM(B.price)` - describes the result after those expressions have been evaluated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    #priceClassifier[A.price][B.price][price]A.priceB.priceSUM(price)SUM(B.price)
    #110-> A#1--10---
    #215-> B#1#2#1, #210152515
    #320-> B#1#2, #3#1, #2, #310204535
    #431-> B#1#2, #3, #4#1, #2, #3, #410317666
    #535#1#2, #3, #4, #5#1, #2, #3, #4, #51035111101
    + +As can be seen in the table, the first row is mapped to pattern variable `A` and subsequent rows are mapped to pattern variable `B`. However, the last row does not fulfill the `B` condition because the sum over all mapped rows `SUM(price)` and the sum over all rows in `B` exceed the specified thresholds. + +### Logical Offsets + +_Logical offsets_ enable navigation within the events that were mapped to a particular pattern variable. This can be expressed +with two corresponding functions: + + + + + + + + + + + + + + + + + + +
    Offset functionsDescription
    + {% highlight text %} +LAST(variable.field, n) +{% endhighlight %} + +

    Returns the value of the field from the event that was mapped to the n-th last element of the variable. The counting starts at the last element mapped.

    +
    + {% highlight text %} +FIRST(variable.field, n) +{% endhighlight %} + +

    Returns the value of the field from the event that was mapped to the n-th element of the variable. The counting starts at the first element mapped.

    +
    + +#### Examples + +For a more thorough example, one can take a look at the following pattern and corresponding conditions: + +{% highlight sql %} +PATTERN (A B+) +DEFINE + A AS A.price > 10, + B AS (LAST(B.price, 1) IS NULL OR B.price > LAST(B.price, 1)) AND + (LAST(B.price, 2) IS NULL OR B.price > 2 * LAST(B.price, 2)) +{% endhighlight %} + +The following table describes how those conditions are evaluated for each incoming event. + +The table consists of the following columns: + * `price` - the price of the incoming row. + * `Classifier` - the classifier of the current row which indicates the pattern variable the row is mapped to. + * `LAST(B.price, 1)`/`LAST(B.price, 2)` - describes the result after those expressions have been evaluated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    priceClassifierLAST(B.price, 1)LAST(B.price, 2)Comment
    10-> A
    15-> BnullnullNotice that LAST(A.price, 1) is null because there is still nothing mapped to B.
    20-> B15null
    31-> B2015
    353120Not mapped because 35 < 2 * 20.
    + +It might also make sense to use the default pattern variable with logical offsets. + +In this case, an offset considers all the rows mapped so far: + +{% highlight sql %} +PATTERN (A B? C) +DEFINE + B AS B.price < 20, + C AS LAST(price, 1) < C.price +{% endhighlight %} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    priceClassifierLAST(price, 1)Comment
    10-> A
    15-> B
    20-> C15LAST(price, 1) is evaluated as the price of the row mapped to the B variable.
    + +If the second row did not map to the `B` variable, we would have the following results: + + + + + + + + + + + + + + + + + + + + + + + + +
    priceClassifierLAST(price, 1)Comment
    10-> A
    20-> C10LAST(price, 1) is evaluated as the price of the row mapped to the A variable.
    + +It is also possible to use multiple pattern variable references in the first argument of the `FIRST/LAST` functions. This way, one can write an expression that accesses multiple columns. +However, all of them must use the same pattern variable. In other words, the value of the `LAST`/`FIRST` function must be computed in a single row. + +Thus, it is possible to use `LAST(A.price * A.tax)`, but an expression like `LAST(A.price * B.tax)` is not allowed. + +After Match Strategy +-------------------- + +The `AFTER MATCH SKIP` clause specifies where to start a new matching procedure after a complete match was found. + +There are four different strategies: +* `SKIP PAST LAST ROW` - resumes the pattern matching at the next row after the last row of the current match. +* `SKIP TO NEXT ROW` - continues searching for a new match starting at the next row after the starting row of the match. +* `SKIP TO LAST variable` - resumes the pattern matching at the last row that is mapped to the specified pattern variable. +* `SKIP TO FIRST variable` - resumes the pattern matching at the first row that is mapped to the specified pattern variable. + +This is also a way to specify how many matches a single event can belong to. For example, with the `SKIP PAST LAST ROW` strategy every event can belong to at most one match. + +#### Examples + +In order to better understand the differences between those strategies one can take a look at the following example. + +For the following input rows: + +{% highlight text %} + symbol tax price rowtime +======== ===== ======= ===================== + XYZ 1 7 2018-09-17 10:00:01 + XYZ 2 9 2018-09-17 10:00:02 + XYZ 1 10 2018-09-17 10:00:03 + XYZ 2 5 2018-09-17 10:00:04 + XYZ 2 17 2018-09-17 10:00:05 + XYZ 2 14 2018-09-17 10:00:06 +{% endhighlight %} + +We evaluate the following query with different strategies: + +{% highlight sql %} +SELECT * +FROM Ticker + MATCH_RECOGNIZE( + PARTITION BY symbol + ORDER BY rowtime + MEASURES + SUM(A.price) AS sumPrice, + FIRST(rowtime) AS startTime, + LAST(rowtime) AS endTime + ONE ROW PER MATCH + [AFTER MATCH STRATEGY] + PATTERN (A+ C) + DEFINE + A AS SUM(A.price) < 30 + ) +{% endhighlight %} + +The query returns the sum of the prices of all rows mapped to `A` and the first and last timestamp of the overall match. + +The query will produce different results based on which `AFTER MATCH` strategy was used: + +##### `AFTER MATCH SKIP PAST LAST ROW` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +The first result matched against the rows #1, #2, #3, #4. + +The second result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO NEXT ROW` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 24 2018-09-17 10:00:02 2018-09-17 10:00:05 + XYZ 15 2018-09-17 10:00:03 2018-09-17 10:00:05 + XYZ 22 2018-09-17 10:00:04 2018-09-17 10:00:06 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +Again, the first result matched against the rows #1, #2, #3, #4. + +Compared to the previous strategy, the next match includes row #2 again for the next matching. Therefore, the second result matched against the rows #2, #3, #4, #5. + +The third result matched against the rows #3, #4, #5. + +The forth result matched against the rows #4, #5, #6. + +The last result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO LAST A` + +{% highlight text %} + symbol sumPrice startTime endTime +======== ========== ===================== ===================== + XYZ 26 2018-09-17 10:00:01 2018-09-17 10:00:04 + XYZ 15 2018-09-17 10:00:03 2018-09-17 10:00:05 + XYZ 22 2018-09-17 10:00:04 2018-09-17 10:00:06 + XYZ 17 2018-09-17 10:00:05 2018-09-17 10:00:06 +{% endhighlight %} + +Again, the first result matched against the rows #1, #2, #3, #4. + +Compared to the previous strategy, the next match includes only row #3 (mapped to `A`) again for the next matching. Therefore, the second result matched against the rows #3, #4, #5. + +The third result matched against the rows #4, #5, #6. + +The last result matched against the rows #5, #6. + +##### `AFTER MATCH SKIP TO FIRST A` + +This combination will produce a runtime exception because one would always try to start a new match where the +last one started. This would produce an infinite loop and, thus, is prohibited. + +One has to keep in mind that in case of the `SKIP TO FIRST/LAST variable` strategy it might be possible that there are no rows mapped to that +variable (e.g. for pattern `A*`). In such cases, a runtime exception will be thrown as the standard requires a valid row to continue the +matching. + +Controlling Memory Consumption +------------------------------ + +Memory consumption is an important consideration when writing `MATCH_RECOGNIZE` queries, as the space of potential matches is built in a breadth-first-like manner. +Having that in mind, one must make sure that the pattern can finish. Preferably with a reasonable number of rows mapped to the match as they have to fit into memory. + +For example, the pattern must not have a quantifier without an upper limit that accepts every single row. Such a pattern could look like this: + +{% highlight sql %} +PATTERN (A B+ C) +DEFINE + A as A.price > 10, + C as C.price > 20 +{% endhighlight %} + +The query will map every incoming row to the `B` variable and thus will never finish. This query could be fixed, e.g., by negating the condition for `C`: + +{% highlight sql %} +PATTERN (A B+ C) +DEFINE + A as A.price > 10, + B as B.price <= 20, + C as C.price > 20 +{% endhighlight %} + +Or by using the [reluctant quantifier](#greedy--reluctant-quantifiers): + +{% highlight sql %} +PATTERN (A B+? C) +DEFINE + A as A.price > 10, + C as C.price > 20 +{% endhighlight %} + +Attention Please note that the `MATCH_RECOGNIZE` clause does not use a configured [state retention time](query_configuration.html#idle-state-retention-time). One may want to use the `WITHIN` [clause](#time-constraint) for this purpose. + +Known Limitations +----------------- + +Flink's implementation of the `MATCH_RECOGNIZE` clause is an ongoing effort, and some features of the SQL standard are not yet supported. + +Unsupported features include: +* Pattern expressions: + * Pattern groups - this means that e.g. quantifiers can not be applied to a subsequence of the pattern. Thus, `(A (B C)+)` is not a valid pattern. + * Alterations - patterns like `PATTERN((A B | C D) E)`, which means that either a subsequence `A B` or `C D` has to be found before looking for the `E` row. + * `PERMUTE` operator - which is equivalent to all permutations of variables that it was applied to e.g. `PATTERN (PERMUTE (A, B, C))` = `PATTERN (A B C | A C B | B A C | B C A | C A B | C B A)`. + * Anchors - `^, $`, which denote beginning/end of a partition, those do not make sense in the streaming context and will not be supported. + * Exclusion - `PATTERN ({- A -} B)` meaning that `A` will be looked for but will not participate in the output. This works only for the `ALL ROWS PER MATCH` mode. + * Reluctant optional quantifier - `PATTERN A??` only the greedy optional quantifier is supported. +* `ALL ROWS PER MATCH` output mode - which produces an output row for every row that participated in the creation of a found match. This also means: + * that the only supported semantic for the `MEASURES` clause is `FINAL` + * `CLASSIFIER` function, which returns the pattern variable that a row was mapped to, is not yet supported. +* `SUBSET` - which allows creating logical groups of pattern variables and using those groups in the `DEFINE` and `MEASURES` clauses. +* Physical offsets - `PREV/NEXT`, which indexes all events seen rather than only those that were mapped to a pattern variable(as in [logical offsets](#logical-offsets) case). +* Extracting time attributes - there is currently no possibility to get a time attribute for subsequent time-based operations. +* `MATCH_RECOGNIZE` is supported only for SQL. There is no equivalent in the Table API. +* Aggregations: + * distinct aggregations are not supported. + +{% top %} diff --git a/docs/dev/table/streaming/query_configuration.md b/docs/dev/table/streaming/query_configuration.md new file mode 100644 index 00000000000000..acfe8e845f1d2b --- /dev/null +++ b/docs/dev/table/streaming/query_configuration.md @@ -0,0 +1,144 @@ +--- +title: "Query Configuration" +nav-parent_id: streaming_tableapi +nav-pos: 6 +--- + + +Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. In many cases, continuous queries on streaming input are capable of computing accurate results that are identical to offline computed results. However, this is not possible in general case because continuous queries have to restrict the size of the state they are maintaining in order to avoid to run out of storage and to be able to process unbounded streaming data over a long period of time. As a result, a continuous query might only be able to provide approximated results depending on the characteristics of the input data and the query itself. + +Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream](../common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](../common.html#emit-a-table). + +
    +
    +{% highlight java %} +StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); +StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); + +// obtain query configuration from TableEnvironment +StreamQueryConfig qConfig = tableEnv.queryConfig(); +// set query parameters +qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(24)); + +// define query +Table result = ... + +// create TableSink +TableSink sink = ... + +// register TableSink +tableEnv.registerTableSink( + "outputTable", // table name + new String[]{...}, // field names + new TypeInformation[]{...}, // field types + sink); // table sink + +// emit result Table via a TableSink +result.insertInto("outputTable", qConfig); + +// convert result Table into a DataStream +DataStream stream = tableEnv.toAppendStream(result, Row.class, qConfig); + +{% endhighlight %} +
    +
    +{% highlight scala %} +val env = StreamExecutionEnvironment.getExecutionEnvironment +val tableEnv = TableEnvironment.getTableEnvironment(env) + +// obtain query configuration from TableEnvironment +val qConfig: StreamQueryConfig = tableEnv.queryConfig +// set query parameters +qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(24)) + +// define query +val result: Table = ??? + +// create TableSink +val sink: TableSink[Row] = ??? + +// register TableSink +tableEnv.registerTableSink( + "outputTable", // table name + Array[String](...), // field names + Array[TypeInformation[_]](...), // field types + sink) // table sink + +// emit result Table via a TableSink +result.insertInto("outputTable", qConfig) + +// convert result Table into a DataStream[Row] +val stream: DataStream[Row] = result.toAppendStream[Row](qConfig) + +{% endhighlight %} +
    +
    + +In the following we describe the parameters of the `QueryConfig` and how they affect the accuracy and resource consumption of a query. + +Idle State Retention Time +------------------------- + +Many queries aggregate or join records on one or more key attributes. When such a query is executed on a stream, the continuous query needs to collect records or maintain partial results per key. If the key domain of the input stream is evolving, i.e., the active key values are changing over time, the continuous query accumulates more and more state as more and more distinct keys are observed. However, often keys become inactive after some time and their corresponding state becomes stale and useless. + +For example the following query computes the number of clicks per session. + +{% highlight sql %} +SELECT sessionId, COUNT(*) FROM clicks GROUP BY sessionId; +{% endhighlight %} + +The `sessionId` attribute is used as a grouping key and the continuous query maintains a count for each `sessionId` it observes. The `sessionId` attribute is evolving over time and `sessionId` values are only active until the session ends, i.e., for a limited period of time. However, the continuous query cannot know about this property of `sessionId` and expects that every `sessionId` value can occur at any point of time. It maintains a count for each observed `sessionId` value. Consequently, the total state size of the query is continuously growing as more and more `sessionId` values are observed. + +The *Idle State Retention Time* parameters define for how long the state of a key is retained without being updated before it is removed. For the previous example query, the count of a `sessionId` would be removed as soon as it has not been updated for the configured period of time. + +By removing the state of a key, the continuous query completely forgets that it has seen this key before. If a record with a key, whose state has been removed before, is processed, the record will be treated as if it was the first record with the respective key. For the example above this means that the count of a `sessionId` would start again at `0`. + +There are two parameters to configure the idle state retention time: +- The *minimum idle state retention time* defines how long the state of an inactive key is at least kept before it is removed. +- The *maximum idle state retention time* defines how long the state of an inactive key is at most kept before it is removed. + +The parameters are specified as follows: + +
    +
    +{% highlight java %} + +StreamQueryConfig qConfig = ... + +// set idle state retention time: min = 12 hours, max = 24 hours +qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(24)); + +{% endhighlight %} +
    +
    +{% highlight scala %} + +val qConfig: StreamQueryConfig = ??? + +// set idle state retention time: min = 12 hours, max = 24 hours +qConfig.withIdleStateRetentionTime(Time.hours(12), Time.hours(24)) + +{% endhighlight %} +
    +
    + +Cleaning up state requires additional bookkeeping which becomes less expensive for larger differences of `minTime` and `maxTime`. The difference between `minTime` and `maxTime` must be at least 5 minutes. + +{% top %} diff --git a/docs/dev/table/streaming/temporal_tables.md b/docs/dev/table/streaming/temporal_tables.md new file mode 100644 index 00000000000000..d511c1f93f4594 --- /dev/null +++ b/docs/dev/table/streaming/temporal_tables.md @@ -0,0 +1,190 @@ +--- +title: "Temporal Tables" +nav-parent_id: streaming_tableapi +nav-pos: 4 +--- + + +Temporal Tables represent a concept of a (parameterized) view on a changing history table that returns the content of a table at a specific point in time. + +Flink can keep track of the changes applied to an underlying append-only table and allows for accessing the table's content at a certain point in time within a query. + +* This will be replaced by the TOC +{:toc} + +Motivation +---------- + +Let's assume that we have the following table `RatesHistory`. + +{% highlight sql %} +SELECT * FROM RatesHistory; + +rowtime currency rate +======= ======== ====== +09:00 US Dollar 102 +09:00 Euro 114 +09:00 Yen 1 +10:45 Euro 116 +11:15 Euro 119 +11:49 Pounds 108 +{% endhighlight %} + +`RatesHistory` represents an ever growing append-only table of currency exchange rates with respect to `Yen` (which has a rate of `1`). +For example, the exchange rate for the period from `09:00` to `10:45` of `Euro` to `Yen` was `114`. From `10:45` to `11:15` it was `116`. + +Given that we would like to output all current rates at the time `10:58`, we would need the following SQL query to compute a result table: + +{% highlight sql %} +SELECT * +FROM RatesHistory AS r +WHERE r.rowtime = ( + SELECT MAX(rowtime) + FROM RatesHistory AS r2 + WHERE r2.currency = r.currency + AND r2.rowtime <= TIME '10:58'); +{% endhighlight %} + +The correlated subquery determines the maximum time for the corresponding currency that is lower or equal than the desired time. The outer query lists the rates that have a maximum timestamp. + +The following table shows the result of such a computation. In our example, the update to `Euro` at `10:45` is taken into account, however, the update to `Euro` at `11:15` and the new entry of `Pounds` are not considered in the table's version at time `10:58`. + +{% highlight text %} +rowtime currency rate +======= ======== ====== +09:00 US Dollar 102 +09:00 Yen 1 +10:45 Euro 116 +{% endhighlight %} + +The concept of *Temporal Tables* aims to simplify such queries, speed up their execution, and reduce Flink's state usage. A *Temporal Table* is a parameterized view on an append-only table that interprets the rows of the append-only table as the changelog of a table and provides the version of that table at a specific point in time. Interpreting the append-only table as a changelog requires the specification of a primary key attribute and a timestamp attribute. The primary key determines which rows are overwritten and the timestamp determines the time during which a row is valid. + +In the above example `currency` would be a primary key for `RatesHistory` table and `rowtime` would be the timestamp attribute. + +In Flink, a temporal table is represented by a *Temporal Table Function*. + +Temporal Table Functions +------------------------ + +In order to access the data in a temporal table, one must pass a [time attribute](time_attributes.html) that determines the version of the table that will be returned. +Flink uses the SQL syntax of [table functions](../udfs.html#table-functions) to provide a way to express it. + +Once defined, a *Temporal Table Function* takes a single time argument `timeAttribute` and returns a set of rows. +This set contains the latest versions of the rows for all of the existing primary keys with respect to the given time attribute. + +Assuming that we defined a temporal table function `Rates(timeAttribute)` based on `RatesHistory` table, we could query such a function in the following way: + +{% highlight sql %} +SELECT * FROM Rates('10:15'); + +rowtime currency rate +======= ======== ====== +09:00 US Dollar 102 +09:00 Euro 114 +09:00 Yen 1 + +SELECT * FROM Rates('11:00'); + +rowtime currency rate +======= ======== ====== +09:00 US Dollar 102 +10:45 Euro 116 +09:00 Yen 1 +{% endhighlight %} + +Each query to `Rates(timeAttribute)` would return the state of the `Rates` for the given `timeAttribute`. + +**Note**: Currently, Flink doesn't support directly querying the temporal table functions with a constant time attribute parameter. At the moment, temporal table functions can only be used in joins. +The example above was used to provide an intuition about what the function `Rates(timeAttribute)` returns. + +See also the page about [joins for continuous queries](joins.html) for more information about how to join with a temporal table. + +### Defining Temporal Table Function + +The following code snippet illustrates how to create a temporal table function from an append-only table. + +
    +
    +{% highlight java %} +import org.apache.flink.table.functions.TemporalTableFunction; +(...) + +// Get the stream and table environments. +StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); +StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); + +// Provide a static data set of the rates history table. +List> ratesHistoryData = new ArrayList<>(); +ratesHistoryData.add(Tuple2.of("US Dollar", 102L)); +ratesHistoryData.add(Tuple2.of("Euro", 114L)); +ratesHistoryData.add(Tuple2.of("Yen", 1L)); +ratesHistoryData.add(Tuple2.of("Euro", 116L)); +ratesHistoryData.add(Tuple2.of("Euro", 119L)); + +// Create and register an example table using above data set. +// In the real setup, you should replace this with your own table. +DataStream> ratesHistoryStream = env.fromCollection(ratesHistoryData); +Table ratesHistory = tEnv.fromDataStream(ratesHistoryStream, "r_currency, r_rate, r_proctime.proctime"); + +tEnv.registerTable("RatesHistory", ratesHistory); + +// Create and register a temporal table function. +// Define "r_proctime" as the time attribute and "r_currency" as the primary key. +TemporalTableFunction rates = ratesHistory.createTemporalTableFunction("r_proctime", "r_currency"); // <==== (1) +tEnv.registerFunction("Rates", rates); // <==== (2) +{% endhighlight %} +
    +
    +{% highlight scala %} +// Get the stream and table environments. +val env = StreamExecutionEnvironment.getExecutionEnvironment +val tEnv = TableEnvironment.getTableEnvironment(env) + +// Provide a static data set of the rates history table. +val ratesHistoryData = new mutable.MutableList[(String, Long)] +ratesHistoryData.+=(("US Dollar", 102L)) +ratesHistoryData.+=(("Euro", 114L)) +ratesHistoryData.+=(("Yen", 1L)) +ratesHistoryData.+=(("Euro", 116L)) +ratesHistoryData.+=(("Euro", 119L)) + +// Create and register an example table using above data set. +// In the real setup, you should replace this with your own table. +val ratesHistory = env + .fromCollection(ratesHistoryData) + .toTable(tEnv, 'r_currency, 'r_rate, 'r_proctime.proctime) + +tEnv.registerTable("RatesHistory", ratesHistory) + +// Create and register TemporalTableFunction. +// Define "r_proctime" as the time attribute and "r_currency" as the primary key. +val rates = ratesHistory.createTemporalTableFunction('r_proctime, 'r_currency) // <==== (1) +tEnv.registerFunction("Rates", rates) // <==== (2) +{% endhighlight %} +
    +
    + +Line `(1)` creates a `rates` [temporal table function](#temporal-table-functions), +which allows us to use the function `rates` in the [Table API](../tableApi.html#joins). + +Line `(2)` registers this function under the name `Rates` in our table environment, +which allows us to use the `Rates` function in [SQL](../sql.html#joins). + +{% top %} diff --git a/docs/dev/table/streaming/time_attributes.md b/docs/dev/table/streaming/time_attributes.md new file mode 100644 index 00000000000000..9014d2a0007f14 --- /dev/null +++ b/docs/dev/table/streaming/time_attributes.md @@ -0,0 +1,351 @@ +--- +title: "Time Attributes" +nav-parent_id: streaming_tableapi +nav-pos: 2 +--- + + +Flink is able to process streaming data based on different notions of *time*. + +- *Processing time* refers to the system time of the machine (also known as "wall-clock time") that is executing the respective operation. +- *Event time* refers to the processing of streaming data based on timestamps which are attached to each row. The timestamps can encode when an event happened. +- *Ingestion time* is the time that events enter Flink; internally, it is treated similarly to event time. + +For more information about time handling in Flink, see the introduction about [Event Time and Watermarks]({{ site.baseurl }}/dev/event_time.html). + +This page explains how time attributes can be defined for time-based operations in Flink's Table API & SQL. + +* This will be replaced by the TOC +{:toc} + +Introduction to Time Attributes +------------------------------- + +Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs. + +Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations. + +As long as a time attribute is not modified and is simply forwarded from one part of the query to another, it remains a valid time attribute. Time attributes behave like regular timestamps and can be accessed for calculations. If a time attribute is used in a calculation, it will be materialized and becomes a regular timestamp. Regular timestamps do not cooperate with Flink's time and watermarking system and thus can not be used for time-based operations anymore. + +Table programs require that the corresponding time characteristic has been specified for the streaming environment: + +
    +
    +{% highlight java %} +final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + +env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); // default + +// alternatively: +// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); +// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); +{% endhighlight %} +
    +
    +{% highlight scala %} +val env = StreamExecutionEnvironment.getExecutionEnvironment + +env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) // default + +// alternatively: +// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) +// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) +{% endhighlight %} +
    +
    + +Processing time +--------------- + +Processing time allows a table program to produce results based on the time of the local machine. It is the simplest notion of time but does not provide determinism. It neither requires timestamp extraction nor watermark generation. + +There are two ways to define a processing time attribute. + +### During DataStream-to-Table Conversion + +The processing time attribute is defined with the `.proctime` property during schema definition. The time attribute must only extend the physical schema by an additional logical field. Thus, it can only be defined at the end of the schema definition. + +
    +
    +{% highlight java %} +DataStream> stream = ...; + +// declare an additional logical field as a processing time attribute +Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.proctime"); + +WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); +{% endhighlight %} +
    +
    +{% highlight scala %} +val stream: DataStream[(String, String)] = ... + +// declare an additional logical field as a processing time attribute +val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'Username, 'Data, 'UserActionTime.proctime) + +val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) +{% endhighlight %} +
    +
    + +### Using a TableSource + +The processing time attribute is defined by a `TableSource` that implements the `DefinedProctimeAttribute` interface. The logical time attribute is appended to the physical schema defined by the return type of the `TableSource`. + +
    +
    +{% highlight java %} +// define a table source with a processing attribute +public class UserActionSource implements StreamTableSource, DefinedProctimeAttribute { + + @Override + public TypeInformation getReturnType() { + String[] names = new String[] {"Username" , "Data"}; + TypeInformation[] types = new TypeInformation[] {Types.STRING(), Types.STRING()}; + return Types.ROW(names, types); + } + + @Override + public DataStream getDataStream(StreamExecutionEnvironment execEnv) { + // create stream + DataStream stream = ...; + return stream; + } + + @Override + public String getProctimeAttribute() { + // field with this name will be appended as a third field + return "UserActionTime"; + } +} + +// register table source +tEnv.registerTableSource("UserActions", new UserActionSource()); + +WindowedTable windowedTable = tEnv + .scan("UserActions") + .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); +{% endhighlight %} +
    +
    +{% highlight scala %} +// define a table source with a processing attribute +class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttribute { + + override def getReturnType = { + val names = Array[String]("Username" , "Data") + val types = Array[TypeInformation[_]](Types.STRING, Types.STRING) + Types.ROW(names, types) + } + + override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = { + // create stream + val stream = ... + stream + } + + override def getProctimeAttribute = { + // field with this name will be appended as a third field + "UserActionTime" + } +} + +// register table source +tEnv.registerTableSource("UserActions", new UserActionSource) + +val windowedTable = tEnv + .scan("UserActions") + .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) +{% endhighlight %} +
    +
    + +Event time +---------- + +Event time allows a table program to produce results based on the time that is contained in every record. This allows for consistent results even in case of out-of-order events or late events. It also ensures replayable results of the table program when reading records from persistent storage. + +Additionally, event time allows for unified syntax for table programs in both batch and streaming environments. A time attribute in a streaming environment can be a regular field of a record in a batch environment. + +In order to handle out-of-order events and distinguish between on-time and late events in streaming, Flink needs to extract timestamps from events and make some kind of progress in time (so-called [watermarks]({{ site.baseurl }}/dev/event_time.html)). + +An event time attribute can be defined either during DataStream-to-Table conversion or by using a TableSource. + +### During DataStream-to-Table Conversion + +The event time attribute is defined with the `.rowtime` property during schema definition. [Timestamps and watermarks]({{ site.baseurl }}/dev/event_time.html) must have been assigned in the `DataStream` that is converted. + +There are two ways of defining the time attribute when converting a `DataStream` into a `Table`. Depending on whether the specified `.rowtime` field name exists in the schema of the `DataStream` or not, the timestamp field is either + +- appended as a new field to the schema or +- replaces an existing field. + +In either case the event time timestamp field will hold the value of the `DataStream` event time timestamp. + +
    +
    +{% highlight java %} + +// Option 1: + +// extract timestamp and assign watermarks based on knowledge of the stream +DataStream> stream = inputStream.assignTimestampsAndWatermarks(...); + +// declare an additional logical field as an event time attribute +Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.rowtime"); + + +// Option 2: + +// extract timestamp from first field, and assign watermarks based on knowledge of the stream +DataStream> stream = inputStream.assignTimestampsAndWatermarks(...); + +// the first field has been used for timestamp extraction, and is no longer necessary +// replace first field with a logical event time attribute +Table table = tEnv.fromDataStream(stream, "UserActionTime.rowtime, Username, Data"); + +// Usage: + +WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); +{% endhighlight %} +
    +
    +{% highlight scala %} + +// Option 1: + +// extract timestamp and assign watermarks based on knowledge of the stream +val stream: DataStream[(String, String)] = inputStream.assignTimestampsAndWatermarks(...) + +// declare an additional logical field as an event time attribute +val table = tEnv.fromDataStream(stream, 'Username, 'Data, 'UserActionTime.rowtime) + + +// Option 2: + +// extract timestamp from first field, and assign watermarks based on knowledge of the stream +val stream: DataStream[(Long, String, String)] = inputStream.assignTimestampsAndWatermarks(...) + +// the first field has been used for timestamp extraction, and is no longer necessary +// replace first field with a logical event time attribute +val table = tEnv.fromDataStream(stream, 'UserActionTime.rowtime, 'Username, 'Data) + +// Usage: + +val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) +{% endhighlight %} +
    +
    + +### Using a TableSource + +The event time attribute is defined by a `TableSource` that implements the `DefinedRowtimeAttributes` interface. The `getRowtimeAttributeDescriptors()` method returns a list of `RowtimeAttributeDescriptor` for describing the final name of a time attribute, a timestamp extractor to derive the values of the attribute, and the watermark strategy associated with the attribute. + +Please make sure that the `DataStream` returned by the `getDataStream()` method is aligned with the defined time attribute. +The timestamps of the `DataStream` (the ones which are assigned by a `TimestampAssigner`) are only considered if a `StreamRecordTimestamp` timestamp extractor is defined. +Watermarks of a `DataStream` are only preserved if a `PreserveWatermarks` watermark strategy is defined. +Otherwise, only the values of the `TableSource`'s rowtime attribute are relevant. + +
    +
    +{% highlight java %} +// define a table source with a rowtime attribute +public class UserActionSource implements StreamTableSource, DefinedRowtimeAttributes { + + @Override + public TypeInformation getReturnType() { + String[] names = new String[] {"Username", "Data", "UserActionTime"}; + TypeInformation[] types = + new TypeInformation[] {Types.STRING(), Types.STRING(), Types.LONG()}; + return Types.ROW(names, types); + } + + @Override + public DataStream getDataStream(StreamExecutionEnvironment execEnv) { + // create stream + // ... + // assign watermarks based on the "UserActionTime" attribute + DataStream stream = inputStream.assignTimestampsAndWatermarks(...); + return stream; + } + + @Override + public List getRowtimeAttributeDescriptors() { + // Mark the "UserActionTime" attribute as event-time attribute. + // We create one attribute descriptor of "UserActionTime". + RowtimeAttributeDescriptor rowtimeAttrDescr = new RowtimeAttributeDescriptor( + "UserActionTime", + new ExistingField("UserActionTime"), + new AscendingTimestamps()); + List listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr); + return listRowtimeAttrDescr; + } +} + +// register the table source +tEnv.registerTableSource("UserActions", new UserActionSource()); + +WindowedTable windowedTable = tEnv + .scan("UserActions") + .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow")); +{% endhighlight %} +
    +
    +{% highlight scala %} +// define a table source with a rowtime attribute +class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttributes { + + override def getReturnType = { + val names = Array[String]("Username" , "Data", "UserActionTime") + val types = Array[TypeInformation[_]](Types.STRING, Types.STRING, Types.LONG) + Types.ROW(names, types) + } + + override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = { + // create stream + // ... + // assign watermarks based on the "UserActionTime" attribute + val stream = inputStream.assignTimestampsAndWatermarks(...) + stream + } + + override def getRowtimeAttributeDescriptors: util.List[RowtimeAttributeDescriptor] = { + // Mark the "UserActionTime" attribute as event-time attribute. + // We create one attribute descriptor of "UserActionTime". + val rowtimeAttrDescr = new RowtimeAttributeDescriptor( + "UserActionTime", + new ExistingField("UserActionTime"), + new AscendingTimestamps) + val listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr) + listRowtimeAttrDescr + } +} + +// register the table source +tEnv.registerTableSource("UserActions", new UserActionSource) + +val windowedTable = tEnv + .scan("UserActions") + .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow) +{% endhighlight %} +
    +
    + +{% top %} diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md index 4e7c544a33acdc..d44eebddf53519 100644 --- a/docs/dev/table/tableApi.md +++ b/docs/dev/table/tableApi.md @@ -24,9 +24,9 @@ under the License. The Table API is a unified, relational API for stream and batch processing. Table API queries can be run on batch or streaming input without modifications. The Table API is a super set of the SQL language and is specially designed for working with Apache Flink. The Table API is a language-integrated API for Scala and Java. Instead of specifying queries as String values as common with SQL, Table API queries are defined in a language-embedded style in Java or Scala with IDE support like autocompletion and syntax validation. -The Table API shares many concepts and parts of its API with Flink's SQL integration. Have a look at the [Common Concepts & API]({{ site.baseurl }}/dev/table/common.html) to learn how to register tables or to create a `Table` object. The [Streaming Concepts]({{ site.baseurl }}/dev/table/streaming.html) page discusses streaming specific concepts such as dynamic tables and time attributes. +The Table API shares many concepts and parts of its API with Flink's SQL integration. Have a look at the [Common Concepts & API]({{ site.baseurl }}/dev/table/common.html) to learn how to register tables or to create a `Table` object. The [Streaming Concepts](./streaming) pages discuss streaming specific concepts such as dynamic tables and time attributes. -The following examples assume a registered table called `Orders` with attributes `(a, b, c, rowtime)`. The `rowtime` field is either a logical [time attribute](streaming.html#time-attributes) in streaming or a regular timestamp field in batch. +The following examples assume a registered table called `Orders` with attributes `(a, b, c, rowtime)`. The `rowtime` field is either a logical [time attribute](./streaming/time_attributes.html) in streaming or a regular timestamp field in batch. * This will be replaced by the TOC {:toc} @@ -59,7 +59,7 @@ Table counts = orders .select("a, b.count as cnt"); // conversion to DataSet -DataSet result = tableEnv.toDataSet(counts, Row.class); +DataSet result = tEnv.toDataSet(counts, Row.class); result.print(); {% endhighlight %} @@ -109,7 +109,7 @@ Table orders = tEnv.scan("Orders"); // schema (a, b, c, rowtime) Table result = orders .filter("a.isNotNull && b.isNotNull && c.isNotNull") - .select("a.lowerCase(), b, rowtime") + .select("a.lowerCase() as a, b, rowtime") .window(Tumble.over("1.hour").on("rowtime").as("hourlyWindow")) .groupBy("hourlyWindow, a") .select("a, hourlyWindow.end as hour, b.avg as avgBillingAmount"); @@ -128,7 +128,7 @@ val orders: Table = tEnv.scan("Orders") // schema (a, b, c, rowtime) val result: Table = orders .filter('a.isNotNull && 'b.isNotNull && 'c.isNotNull) - .select('a.lowerCase(), 'b, 'rowtime) + .select('a.lowerCase() as 'a, 'b, 'rowtime) .window(Tumble over 1.hour on 'rowtime as 'hourlyWindow) .groupBy('hourlyWindow, 'a) .select('a, 'hourlyWindow.end as 'hour, 'b.avg as 'avgBillingAmount) @@ -137,7 +137,7 @@ val result: Table = orders
    -Since the Table API is a unified API for batch and streaming data, both example programs can be executed on batch and streaming inputs without any modification of the table program itself. In both cases, the program produces the same results given that streaming records are not late (see [Streaming Concepts](streaming.html) for details). +Since the Table API is a unified API for batch and streaming data, both example programs can be executed on batch and streaming inputs without any modification of the table program itself. In both cases, the program produces the same results given that streaming records are not late (see [Streaming Concepts](streaming) for details). {% top %} @@ -329,7 +329,7 @@ val result = orders.where('b === "red") Table orders = tableEnv.scan("Orders"); Table result = orders.groupBy("a").select("a, b.sum as d"); {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the type of aggregation and the number of distinct grouping keys. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the type of aggregation and the number of distinct grouping keys. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -367,7 +367,45 @@ Table result = orders .as("w")) .select("a, b.avg over w, b.max over w, b.min over w"); // sliding aggregate {% endhighlight %} -

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute.

    +

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute.

    + + + + + Distinct Aggregation
    + Batch Streaming
    + Result Updating + + +

    Similar to a SQL DISTINCT aggregation clause such as COUNT(DISTINCT a). Distinct aggregation declares that an aggregation function (built-in or user-defined) is only applied on distinct input values. Distinct can be applied to GroupBy Aggregation, GroupBy Window Aggregation and Over Window Aggregation.

    +{% highlight java %} +Table orders = tableEnv.scan("Orders"); +// Distinct aggregation on group by +Table groupByDistinctResult = orders + .groupBy("a") + .select("a, b.sum.distinct as d"); +// Distinct aggregation on time window group by +Table groupByWindowDistinctResult = orders + .window(Tumble.over("5.minutes").on("rowtime").as("w")).groupBy("a, w") + .select("a, b.sum.distinct as d"); +// Distinct aggregation on over window +Table result = orders + .window(Over + .partitionBy("a") + .orderBy("rowtime") + .preceding("UNBOUNDED_RANGE") + .as("w")) + .select("a, b.avg.distinct over w, b.max over w, b.min over w"); +{% endhighlight %} +

    User-defined aggregation function can also be used with DISTINCT modifiers. To calculate the aggregate results only for distinct values, simply add the distinct modifier towards the aggregation function.

    +{% highlight java %} +Table orders = tEnv.scan("Orders"); + +// Use distinct aggregation for user-defined aggregate functions +tEnv.registerFunction("myUdagg", new MyUdagg()); +orders.groupBy("users").select("users, myUdagg.distinct(points) as myDistinctResult"); +{% endhighlight %} +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -382,7 +420,7 @@ Table result = orders Table orders = tableEnv.scan("Orders"); Table result = orders.distinct(); {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -412,7 +450,7 @@ Table result = orders.distinct(); val orders: Table = tableEnv.scan("Orders") val result = orders.groupBy('a).select('a, 'b.sum as 'd) {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the type of aggregation and the number of distinct grouping keys. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the type of aggregation and the number of distinct grouping keys. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -450,7 +488,45 @@ val result: Table = orders as 'w) .select('a, 'b.avg over 'w, 'b.max over 'w, 'b.min over 'w) // sliding aggregate {% endhighlight %} -

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute.

    +

    Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute.

    + + + + + Distinct Aggregation
    + Batch Streaming
    + Result Updating + + +

    Similar to a SQL DISTINCT AGGREGATION clause such as COUNT(DISTINCT a). Distinct aggregation declares that an aggregation function (built-in or user-defined) is only applied on distinct input values. Distinct can be applied to GroupBy Aggregation, GroupBy Window Aggregation and Over Window Aggregation.

    +{% highlight scala %} +val orders: Table = tableEnv.scan("Orders"); +// Distinct aggregation on group by +val groupByDistinctResult = orders + .groupBy('a) + .select('a, 'b.sum.distinct as 'd) +// Distinct aggregation on time window group by +val groupByWindowDistinctResult = orders + .window(Tumble over 5.minutes on 'rowtime as 'w).groupBy('a, 'w) + .select('a, 'b.sum.distinct as 'd) +// Distinct aggregation on over window +val result = orders + .window(Over + partitionBy 'a + orderBy 'rowtime + preceding UNBOUNDED_RANGE + as 'w) + .select('a, 'b.avg.distinct over 'w, 'b.max over 'w, 'b.min over 'w) +{% endhighlight %} +

    User-defined aggregation function can also be used with DISTINCT modifiers. To calculate the aggregate results only for distinct values, simply add the distinct modifier towards the aggregation function.

    +{% highlight scala %} +val orders: Table = tEnv.scan("Orders"); + +// Use distinct aggregation for user-defined aggregate functions +val myUdagg = new MyUdagg(); +orders.groupBy('users).select('users, myUdagg.distinct('points) as 'myDistinctResult); +{% endhighlight %} +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -464,7 +540,7 @@ val result: Table = orders val orders: Table = tableEnv.scan("Orders") val result = orders.distinct() {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -500,14 +576,16 @@ Table left = tableEnv.fromDataSet(ds1, "a, b, c"); Table right = tableEnv.fromDataSet(ds2, "d, e, f"); Table result = left.join(right).where("a = d").select("a, b, e"); {% endhighlight %} -

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Streaming Concepts for details.

    +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    - Outer Joins
    + Outer Join
    Batch + Streaming + Result Updating

    Similar to SQL LEFT/RIGHT/FULL OUTER JOIN clauses. Joins two tables. Both tables must have distinct field names and at least one equality join predicate must be defined.

    @@ -519,6 +597,7 @@ Table leftOuterResult = left.leftOuterJoin(right, "a = d").select("a, b, e"); Table rightOuterResult = left.rightOuterJoin(right, "a = d").select("a, b, e"); Table fullOuterResult = left.fullOuterJoin(right, "a = d").select("a, b, e"); {% endhighlight %} +

    Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -529,7 +608,7 @@ Table fullOuterResult = left.fullOuterJoin(right, "a = d").select("a, b, e");

    Note: Time-windowed joins are a subset of regular joins that can be processed in a streaming fashion.

    -

    A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >) or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

    +

    A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >) or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

    For example, the following predicates are valid window join conditions:

      @@ -549,47 +628,74 @@ Table result = left.join(right) - TableFunction Inner Join
      + Inner Join with Table Function
      Batch Streaming -

      Joins a table with a the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. A row of the left (outer) table is dropped, if its table function call returns an empty result. +

      Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. A row of the left (outer) table is dropped, if its table function call returns an empty result.

      {% highlight java %} -// register function +// register User-Defined Table Function TableFunction split = new MySplitUDTF(); -tEnv.registerFunction("split", split); +tableEnv.registerFunction("split", split); // join Table orders = tableEnv.scan("Orders"); Table result = orders - .join(new Table(tEnv, "split(c)").as("s", "t", "v")) + .join(new Table(tableEnv, "split(c)").as("s", "t", "v")) .select("a, b, s, t, v"); {% endhighlight %} - TableFunction Left Outer Join
      + Left Outer Join with Table Function
      Batch Streaming -

      Joins a table with a the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. If a table function call returns an empty result, the corresponding outer row is preserved and the result padded with null values. +

      Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. If a table function call returns an empty result, the corresponding outer row is preserved and the result padded with null values.

      Note: Currently, the predicate of a table function left outer join can only be empty or literal true.

      -

      {% highlight java %} -// register function +// register User-Defined Table Function TableFunction split = new MySplitUDTF(); -tEnv.registerFunction("split", split); +tableEnv.registerFunction("split", split); // join Table orders = tableEnv.scan("Orders"); Table result = orders - .leftOuterJoin(new Table(tEnv, "split(c)").as("s", "t", "v")) + .leftOuterJoin(new Table(tableEnv, "split(c)").as("s", "t", "v")) .select("a, b, s, t, v"); {% endhighlight %} + + + Join with Temporal Table
      + Streaming + + +

      Temporal tables are tables that track changes over time.

      +

      A temporal table function provides access to the state of a temporal table at a specific point in time. + The syntax to join a table with a temporal table function is the same as in Inner Join with Table Function.

      + +

      Currently only inner joins with temporal tables are supported.

      +{% highlight java %} +Table ratesHistory = tableEnv.scan("RatesHistory"); + +// register temporal table function with a time attribute and primary key +TemporalTableFunction rates = ratesHistory.createTemporalTableFunction( + "r_proctime", + "r_currency"); +tableEnv.registerFunction("rates", rates); + +// join with "Orders" based on the time attribute and key +Table orders = tableEnv.scan("Orders"); +Table result = orders + .join(new Table(tEnv, "rates(o_proctime)"), "o_currency = r_currency") +{% endhighlight %} +

      For more information please check the more detailed temporal tables concept description.

      + + @@ -610,6 +716,7 @@ Table result = orders Inner Join
      Batch + Streaming

      Similar to a SQL JOIN clause. Joins two tables. Both tables must have distinct field names and at least one equality join predicate must be defined through join operator or using a where or filter operator.

      @@ -618,12 +725,15 @@ val left = ds1.toTable(tableEnv, 'a, 'b, 'c) val right = ds2.toTable(tableEnv, 'd, 'e, 'f) val result = left.join(right).where('a === 'd).select('a, 'b, 'e) {% endhighlight %} +

      Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

      - Outer Joins
      + Outer Join
      Batch + Streaming + Result Updating

      Similar to SQL LEFT/RIGHT/FULL OUTER JOIN clauses. Joins two tables. Both tables must have distinct field names and at least one equality join predicate must be defined.

      @@ -635,6 +745,7 @@ val leftOuterResult = left.leftOuterJoin(right, 'a === 'd).select('a, 'b, 'e) val rightOuterResult = left.rightOuterJoin(right, 'a === 'd).select('a, 'b, 'e) val fullOuterResult = left.fullOuterJoin(right, 'a === 'd).select('a, 'b, 'e) {% endhighlight %} +

      Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

      @@ -645,7 +756,7 @@ val fullOuterResult = left.fullOuterJoin(right, 'a === 'd).select('a, 'b, 'e)

      Note: Time-windowed joins are a subset of regular joins that can be processed in a streaming fashion.

      -

      A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >) or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

      +

      A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >) or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.

      For example, the following predicates are valid window join conditions:

        @@ -665,14 +776,14 @@ val result = left.join(right) - TableFunction Inner Join
        + Inner Join with Table Function
        Batch Streaming -

        Joins a table with a the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. A row of the left (outer) table is dropped, if its table function call returns an empty result. +

        Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. A row of the left (outer) table is dropped, if its table function call returns an empty result.

        {% highlight scala %} -// instantiate function +// instantiate User-Defined Table Function val split: TableFunction[_] = new MySplitUDTF() // join @@ -684,14 +795,13 @@ val result: Table = table - TableFunction Left Outer Join
        + Left Outer Join with Table Function
        Batch Streaming -

        Joins a table with a the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. If a table function call returns an empty result, the corresponding outer row is preserved and the result padded with null values. +

        Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function. If a table function call returns an empty result, the corresponding outer row is preserved and the result padded with null values.

        Note: Currently, the predicate of a table function left outer join can only be empty or literal true.

        -

        {% highlight scala %} -// instantiate function +// instantiate User-Defined Table Function val split: TableFunction[_] = new MySplitUDTF() // join @@ -702,6 +812,32 @@ val result: Table = table + + + Join with Temporal Table
        + Streaming + + +

        Temporal tables are tables that track their changes over time.

        +

        A temporal table function provides access to the state of a temporal table at a specific point in time. + The syntax to join a table with a temporal table function is the same as in Inner Join with Table Function.

        + +

        Currently only inner joins with temporal tables are supported.

        +{% highlight scala %} +val ratesHistory = tableEnv.scan("RatesHistory") + +// register temporal table function with a time attribute and primary key +val rates = ratesHistory.createTemporalTableFunction('r_proctime, 'r_currency) + +// join with "Orders" based on the time attribute and key +val orders = tableEnv.scan("Orders") +val result = orders + .join(rates('o_rowtime), 'r_currency === 'o_currency) +{% endhighlight %} +

        For more information please check the more detailed temporal tables concept description.

        + + +
    @@ -815,7 +951,7 @@ Table result = left.minusAll(right); In
    - Batch + Batch Streaming

    Similar to a SQL IN clause. In returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression.

    @@ -830,6 +966,8 @@ Table result = left.select("a, b, c").where("a.in(" + right + ")"); tableEnv.registerTable("RightTable", right); Table result = left.select("a, b, c").where("a.in(RightTable)"); {% endhighlight %} + +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -940,7 +1078,7 @@ val result = left.minusAll(right) In
    - Batch + Batch Streaming

    Similar to a SQL IN clause. In returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression.

    @@ -949,6 +1087,7 @@ val left = ds1.toTable(tableEnv, 'a, 'b, 'c) val right = ds2.toTable(tableEnv, 'a) val result = left.select('a, 'b, 'c).where('a.in(right)) {% endhighlight %} +

    Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.

    @@ -1178,7 +1317,7 @@ val table = input
    -Window properties such as the start, end, or rowtime timestamp of a time window can be added in the select statement as a property of the window alias as `w.start`, `w.end`, and `w.rowtime`, respectively. The window start and rowtime timestamps are the inclusive lower and uppper window boundaries. In contrast, the window end timestamp is the exclusive upper window boundary. For example a tumbling window of 30 minutes that starts at 2pm would have `14:00:00.000` as start timestamp, `14:29:59.999` as rowtime timestamp, and `14:30:00.000` as end timestamp. +Window properties such as the start, end, or rowtime timestamp of a time window can be added in the select statement as a property of the window alias as `w.start`, `w.end`, and `w.rowtime`, respectively. The window start and rowtime timestamps are the inclusive lower and upper window boundaries. In contrast, the window end timestamp is the exclusive upper window boundary. For example a tumbling window of 30 minutes that starts at 2pm would have `14:00:00.000` as start timestamp, `14:29:59.999` as rowtime timestamp, and `14:30:00.000` as end timestamp.
    @@ -1223,7 +1362,7 @@ Tumbling windows are defined by using the `Tumble` class as follows: on - The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. + The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. as @@ -1285,7 +1424,7 @@ Sliding windows are defined by using the `Slide` class as follows: on - The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. + The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. as @@ -1343,7 +1482,7 @@ A session window is defined by using the `Session` class as follows: on - The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. + The time attribute to group (time interval) or sort (row count) on. For batch queries this might be any Long or Timestamp attribute. For streaming queries this must be a declared event-time or processing-time time attribute. as @@ -1427,18 +1566,20 @@ The `OverWindow` defines a range of rows over which aggregates are computed. `Ov

    Defines the order of rows within each partition and thereby the order in which the aggregate functions are applied to rows.

    -

    Note: For streaming queries this must be a declared event-time or processing-time time attribute. Currently, only a single sort attribute is supported.

    +

    Note: For streaming queries this must be a declared event-time or processing-time time attribute. Currently, only a single sort attribute is supported.

    preceding - Required + Optional

    Defines the interval of rows that are included in the window and precede the current row. The interval can either be specified as time or row-count interval.

    Bounded over windows are specified with the size of the interval, e.g., 10.minutes for a time interval or 10.rows for a row-count interval.

    Unbounded over windows are specified using a constant, i.e., UNBOUNDED_RANGE for a time interval or UNBOUNDED_ROW for a row-count interval. Unbounded over windows start with the first row of a partition.

    + +

    If the preceding clause is omitted, UNBOUNDED_RANGE and CURRENT_RANGE are used as the default preceding and following for the window.

    @@ -1566,8 +1707,13 @@ The Table API is built on top of Flink's DataSet and DataStream APIs. Internally | `Types.OBJECT_ARRAY` | `ARRAY` | e.g. `java.lang.Byte[]`| | `Types.MAP` | `MAP` | `java.util.HashMap` | | `Types.MULTISET` | `MULTISET` | e.g. `java.util.HashMap` for a multiset of `String` | +| `Types.ROW` | `ROW` | `org.apache.flink.types.Row` | + +Generic types and (nested) composite types (e.g., POJOs, tuples, rows, Scala case classes) can be fields of a row as well. + +Fields of composite types with arbitrary nesting can be accessed with [value access functions](functions.html#value-access-functions). -Generic types and composite types (e.g., POJOs or Tuples) can be fields of a row as well. Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html). Composite types can be accessed with [built-in functions](#built-in-functions) (see *Value access functions* section). +Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html). {% top %} @@ -1582,7 +1728,7 @@ This is the EBNF grammar for expressions: expressionList = expression , { "," , expression } ; -expression = timeIndicator | overConstant | alias ; +expression = overConstant | alias ; alias = logic | ( logic , "as" , fieldReference ) | ( logic , "as" , "(" , fieldReference , { "," , fieldReference } , ")" ) ; @@ -1594,27 +1740,41 @@ term = product , [ ( "+" | "-" ) , product ] ; product = unary , [ ( "*" | "/" | "%") , unary ] ; -unary = [ "!" | "-" ] , composite ; +unary = [ "!" | "-" | "+" ] , composite ; -composite = over | nullLiteral | suffixed | atom ; +composite = over | suffixed | nullLiteral | prefixed | atom ; -suffixed = interval | cast | as | if | functionCall ; +suffixed = interval | suffixAs | suffixCast | suffixIf | suffixDistinct | suffixFunctionCall | timeIndicator ; + +prefixed = prefixAs | prefixCast | prefixIf | prefixDistinct | prefixFunctionCall ; interval = timeInterval | rowInterval ; -timeInterval = composite , "." , ("year" | "years" | "month" | "months" | "day" | "days" | "hour" | "hours" | "minute" | "minutes" | "second" | "seconds" | "milli" | "millis") ; +timeInterval = composite , "." , ("year" | "years" | "quarter" | "quarters" | "month" | "months" | "week" | "weeks" | "day" | "days" | "hour" | "hours" | "minute" | "minutes" | "second" | "seconds" | "milli" | "millis") ; rowInterval = composite , "." , "rows" ; -cast = composite , ".cast(" , dataType , ")" ; +suffixCast = composite , ".cast(" , dataType , ")" ; + +prefixCast = "cast(" , expression , dataType , ")" ; dataType = "BYTE" | "SHORT" | "INT" | "LONG" | "FLOAT" | "DOUBLE" | "BOOLEAN" | "STRING" | "DECIMAL" | "SQL_DATE" | "SQL_TIME" | "SQL_TIMESTAMP" | "INTERVAL_MONTHS" | "INTERVAL_MILLIS" | ( "MAP" , "(" , dataType , "," , dataType , ")" ) | ( "PRIMITIVE_ARRAY" , "(" , dataType , ")" ) | ( "OBJECT_ARRAY" , "(" , dataType , ")" ) ; -as = composite , ".as(" , fieldReference , ")" ; +suffixAs = composite , ".as(" , fieldReference , ")" ; + +prefixAs = "as(" , expression, fieldReference , ")" ; + +suffixIf = composite , ".?(" , expression , "," , expression , ")" ; -if = composite , ".?(" , expression , "," , expression , ")" ; +prefixIf = "?(" , expression , "," , expression , "," , expression , ")" ; -functionCall = composite , "." , functionIdentifier , [ "(" , [ expression , { "," , expression } ] , ")" ] ; +suffixDistinct = composite , "distinct.()" ; + +prefixDistinct = functionIdentifier , ".distinct" , [ "(" , [ expression , { "," , expression } ] , ")" ] ; + +suffixFunctionCall = composite , "." , functionIdentifier , [ "(" , [ expression , { "," , expression } ] , ")" ] ; + +prefixFunctionCall = functionIdentifier , [ "(" , [ expression , { "," , expression } ] , ")" ] ; atom = ( "(" , expression , ")" ) | literal | fieldReference ; @@ -1622,7 +1782,7 @@ fieldReference = "*" | identifier ; nullLiteral = "Null(" , dataType , ")" ; -timeIntervalUnit = "YEAR" | "YEAR_TO_MONTH" | "MONTH" | "DAY" | "DAY_TO_HOUR" | "DAY_TO_MINUTE" | "DAY_TO_SECOND" | "HOUR" | "HOUR_TO_MINUTE" | "HOUR_TO_SECOND" | "MINUTE" | "MINUTE_TO_SECOND" | "SECOND" ; +timeIntervalUnit = "YEAR" | "YEAR_TO_MONTH" | "MONTH" | "QUARTER" | "WEEK" | "DAY" | "DAY_TO_HOUR" | "DAY_TO_MINUTE" | "DAY_TO_SECOND" | "HOUR" | "HOUR_TO_MINUTE" | "HOUR_TO_SECOND" | "MINUTE" | "MINUTE_TO_SECOND" | "SECOND" ; timePointUnit = "YEAR" | "MONTH" | "DAY" | "HOUR" | "MINUTE" | "SECOND" | "QUARTER" | "WEEK" | "MILLISECOND" | "MICROSECOND" ; @@ -1634,8 +1794,11 @@ timeIndicator = fieldReference , "." , ( "proctime" | "rowtime" ) ; {% endhighlight %} -Here, `literal` is a valid Java literal, `fieldReference` specifies a column in the data (or all columns if `*` is used), and `functionIdentifier` specifies a supported scalar function. The -column names and function names follow Java identifier syntax. Expressions specified as Strings can also use prefix notation instead of suffix notation to call operators and functions. +Here, `literal` is a valid Java literal. String literals can be specified using single or double quotes. Duplicate the quote for escaping (e.g. `'It''s me.'` or `"I ""like"" dogs."`). + +The `fieldReference` specifies a column in the data (or all columns if `*` is used), and `functionIdentifier` specifies a supported scalar function. The column names and function names follow Java identifier syntax. + +Expressions specified as strings can also use prefix notation instead of suffix notation to call operators and functions. If working with exact numeric values or large decimals is required, the Table API also supports Java's BigDecimal type. In the Scala Table API decimals can be defined by `BigDecimal("123456")` and in Java by appending a "p" for precise e.g. `123456p`. @@ -1644,2936 +1807,3 @@ In order to work with temporal values the Table API supports Java SQL's Date, Ti Temporal intervals can be represented as number of months (`Types.INTERVAL_MONTHS`) or number of milliseconds (`Types.INTERVAL_MILLIS`). Intervals of same type can be added or subtracted (e.g. `1.hour + 10.minutes`). Intervals of milliseconds can be added to time points (e.g. `"2016-08-10".toDate + 5.days`). {% top %} - -Built-In Functions ------------------- - -The Table API comes with a set of built-in functions for data transformations. This section gives a brief overview of the available functions. - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Comparison functionsDescription
    - {% highlight java %} -ANY === ANY -{% endhighlight %} - -

    Equals.

    -
    - {% highlight java %} -ANY !== ANY -{% endhighlight %} - -

    Not equal.

    -
    - {% highlight java %} -ANY > ANY -{% endhighlight %} - -

    Greater than.

    -
    - {% highlight java %} -ANY >= ANY -{% endhighlight %} - -

    Greater than or equal.

    -
    - {% highlight java %} -ANY < ANY -{% endhighlight %} - -

    Less than.

    -
    - {% highlight java %} -ANY <= ANY -{% endhighlight %} - -

    Less than or equal.

    -
    - {% highlight java %} -ANY.isNull -{% endhighlight %} - -

    Returns true if the given expression is null.

    -
    - {% highlight java %} -ANY.isNotNull -{% endhighlight %} - -

    Returns true if the given expression is not null.

    -
    - {% highlight java %} -STRING.like(STRING) -{% endhighlight %} - -

    Returns true, if a string matches the specified LIKE pattern. E.g. "Jo_n%" matches all strings that start with "Jo(arbitrary letter)n".

    -
    - {% highlight java %} -STRING.similar(STRING) -{% endhighlight %} - -

    Returns true, if a string matches the specified SQL regex pattern. E.g. "A+" matches all strings that consist of at least one "A".

    -
    - {% highlight java %} -ANY.in(ANY, ANY, ...) -{% endhighlight %} - -

    Returns true if an expression exists in a given list of expressions. This is a shorthand for multiple OR conditions. If the testing set contains null, the result will be null if the element can not be found and true if it can be found. If element is null, the result is always null. E.g. "42.in(1, 2, 3)" leads to false.

    -
    - {% highlight java %} -ANY.in(TABLE) -{% endhighlight %} - -

    Returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression. Note: This operation is not supported in a streaming environment yet.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Logical functionsDescription
    - {% highlight java %} -boolean1 || boolean2 -{% endhighlight %} - -

    Returns true if boolean1 is true or boolean2 is true. Supports three-valued logic.

    -
    - {% highlight java %} -boolean1 && boolean2 -{% endhighlight %} - -

    Returns true if boolean1 and boolean2 are both true. Supports three-valued logic.

    -
    - {% highlight java %} -!BOOLEAN -{% endhighlight %} - -

    Returns true if boolean expression is not true; returns null if boolean is null.

    -
    - {% highlight java %} -BOOLEAN.isTrue -{% endhighlight %} - -

    Returns true if the given boolean expression is true. False otherwise (for null and false).

    -
    - {% highlight java %} -BOOLEAN.isFalse -{% endhighlight %} - -

    Returns true if given boolean expression is false. False otherwise (for null and true).

    -
    - {% highlight java %} -BOOLEAN.isNotTrue -{% endhighlight %} - -

    Returns true if the given boolean expression is not true (for null and false). False otherwise.

    -
    - {% highlight java %} -BOOLEAN.isNotFalse -{% endhighlight %} - -

    Returns true if given boolean expression is not false (for null and true). False otherwise.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Arithmetic functionsDescription
    - {% highlight java %} -+ numeric -{% endhighlight %} - -

    Returns numeric.

    -
    - {% highlight java %} -- numeric -{% endhighlight %} - -

    Returns negative numeric.

    -
    - {% highlight java %} -numeric1 + numeric2 -{% endhighlight %} - -

    Returns numeric1 plus numeric2.

    -
    - {% highlight java %} -numeric1 - numeric2 -{% endhighlight %} - -

    Returns numeric1 minus numeric2.

    -
    - {% highlight java %} -numeric1 * numeric2 -{% endhighlight %} - -

    Returns numeric1 multiplied by numeric2.

    -
    - {% highlight java %} -numeric1 / numeric2 -{% endhighlight %} - -

    Returns numeric1 divided by numeric2.

    -
    - {% highlight java %} -numeric1.power(numeric2) -{% endhighlight %} - -

    Returns numeric1 raised to the power of numeric2.

    -
    - {% highlight java %} -NUMERIC.abs() -{% endhighlight %} - -

    Calculates the absolute value of given value.

    -
    - {% highlight java %} -numeric1 % numeric2 -{% endhighlight %} - -

    Returns the remainder (modulus) of numeric1 divided by numeric2. The result is negative only if numeric1 is negative.

    -
    - {% highlight java %} -NUMERIC.sqrt() -{% endhighlight %} - -

    Calculates the square root of a given value.

    -
    - {% highlight java %} -NUMERIC.ln() -{% endhighlight %} - -

    Calculates the natural logarithm of given value.

    -
    - {% highlight java %} -NUMERIC.log10() -{% endhighlight %} - -

    Calculates the base 10 logarithm of given value.

    -
    - {% highlight java %} -NUMERIC.exp() -{% endhighlight %} - -

    Calculates the Euler's number raised to the given power.

    -
    - {% highlight java %} -NUMERIC.ceil() -{% endhighlight %} - -

    Calculates the smallest integer greater than or equal to a given number.

    -
    - {% highlight java %} -NUMERIC.floor() -{% endhighlight %} - -

    Calculates the largest integer less than or equal to a given number.

    -
    - {% highlight java %} -NUMERIC.sin() -{% endhighlight %} - -

    Calculates the sine of a given number.

    -
    - {% highlight java %} -NUMERIC.cos() -{% endhighlight %} - -

    Calculates the cosine of a given number.

    -
    - {% highlight java %} -NUMERIC.tan() -{% endhighlight %} - -

    Calculates the tangent of a given number.

    -
    - {% highlight java %} -NUMERIC.cot() -{% endhighlight %} - -

    Calculates the cotangent of a given number.

    -
    - {% highlight java %} -NUMERIC.asin() -{% endhighlight %} - -

    Calculates the arc sine of a given number.

    -
    - {% highlight java %} -NUMERIC.acos() -{% endhighlight %} - -

    Calculates the arc cosine of a given number.

    -
    - {% highlight java %} -NUMERIC.atan() -{% endhighlight %} - -

    Calculates the arc tangent of a given number.

    -
    - {% highlight java %} -NUMERIC.degrees() -{% endhighlight %} - -

    Converts numeric from radians to degrees.

    -
    - {% highlight java %} -NUMERIC.radians() -{% endhighlight %} - -

    Converts numeric from degrees to radians.

    -
    - {% highlight java %} -NUMERIC.sign() -{% endhighlight %} - -

    Calculates the signum of a given number.

    -
    - {% highlight java %} -NUMERIC.round(INT) -{% endhighlight %} - -

    Rounds the given number to integer places right to the decimal point.

    -
    - {% highlight java %} -pi() -{% endhighlight %} - -

    Returns a value that is closer than any other value to pi.

    -
    - {% highlight java %} -e() -{% endhighlight %} - -

    Returns a value that is closer than any other value to e.

    -
    - {% highlight java %} -rand() -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    -
    - {% highlight java %} -rand(seed integer) -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with a initial seed. Two rand functions will return identical sequences of numbers if they have same initial seed.

    -
    - {% highlight java %} -randInteger(bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive).

    -
    - {% highlight java %} -randInteger(seed integer, bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive) with a initial seed. Two randInteger functions will return identical sequences of numbers if they have same initial seed and same bound.

    -
    - {% highlight java %} -NUMERIC.bin() -{% endhighlight %} - -

    Returns a string representation of an integer numeric value in binary format. Returns null if numeric is null. E.g. "4" leads to "100", "12" leads to "1100".

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    String functionsDescription
    - {% highlight java %} -STRING + STRING -{% endhighlight %} - -

    Concatenates two character strings.

    -
    - {% highlight java %} -STRING.charLength() -{% endhighlight %} - -

    Returns the length of a String.

    -
    - {% highlight java %} -STRING.upperCase() -{% endhighlight %} - -

    Returns all of the characters in a string in upper case using the rules of the default locale.

    -
    - {% highlight java %} -STRING.lowerCase() -{% endhighlight %} - -

    Returns all of the characters in a string in lower case using the rules of the default locale.

    -
    - {% highlight java %} -STRING.position(STRING) -{% endhighlight %} - -

    Returns the position of string in an other string starting at 1. Returns 0 if string could not be found. E.g. 'a'.position('bbbbba') leads to 6.

    -
    - {% highlight java %} -STRING.trim(LEADING, STRING) -STRING.trim(TRAILING, STRING) -STRING.trim(BOTH, STRING) -STRING.trim(BOTH) -STRING.trim() -{% endhighlight %} - -

    Removes leading and/or trailing characters from the given string. By default, whitespaces at both sides are removed.

    -
    - {% highlight java %} -STRING.overlay(STRING, INT) -STRING.overlay(STRING, INT, INT) -{% endhighlight %} - -

    Replaces a substring of string with a string starting at a position (starting at 1). An optional length specifies how many characters should be removed. E.g. 'xxxxxtest'.overlay('xxxx', 6) leads to "xxxxxxxxx", 'xxxxxtest'.overlay('xxxx', 6, 2) leads to "xxxxxxxxxst".

    -
    - {% highlight java %} -STRING.substring(INT) -{% endhighlight %} - -

    Creates a substring of the given string beginning at the given index to the end. The start index starts at 1 and is inclusive.

    -
    - {% highlight java %} -STRING.substring(INT, INT) -{% endhighlight %} - -

    Creates a substring of the given string at the given index for the given length. The index starts at 1 and is inclusive, i.e., the character at the index is included in the substring. The substring has the specified length or less.

    -
    - {% highlight java %} -STRING.initCap() -{% endhighlight %} - -

    Converts the initial letter of each word in a string to uppercase. Assumes a string containing only [A-Za-z0-9], everything else is treated as whitespace.

    -
    - {% highlight java %} -STRING.lpad(len INT, pad STRING) -{% endhighlight %} - -

    Returns a string left-padded with the given pad string to a length of len characters. If the string is longer than len, the return value is shortened to len characters. E.g. "hi".lpad(4, '??') returns "??hi", "hi".lpad(1, '??') returns "h".

    -
    - {% highlight java %} -STRING.rpad(len INT, pad STRING) -{% endhighlight %} - -

    Returns a string right-padded with the given pad string to a length of len characters. If the string is longer than len, the return value is shortened to len characters. E.g. "hi".rpad(4, '??') returns "hi??", "hi".rpad(1, '??') returns "h".

    -
    - {% highlight text %} -concat(string1, string2,...) -{% endhighlight %} - -

    Returns the string that results from concatenating the arguments. Returns NULL if any argument is NULL. E.g. concat("AA", "BB", "CC") returns AABBCC.

    -
    - {% highlight text %} -concat_ws(separator, string1, string2,...) -{% endhighlight %} - -

    Returns the string that results from concatenating the arguments using a separator. The separator is added between the strings to be concatenated. Returns NULL If the separator is NULL. concat_ws() does not skip empty strings. However, it does skip any NULL argument. E.g. concat_ws("~", "AA", "BB", "", "CC") returns AA~BB~~CC

    -
    - - - - - - - - - - - - - - - - - -
    Conditional functionsDescription
    - {% highlight java %} -BOOLEAN.?(value1, value2) -{% endhighlight %} - -

    Ternary conditional operator that decides which of two other expressions should be evaluated based on a evaluated boolean condition. E.g. (42 > 5).?("A", "B") leads to "A".

    -
    - - - - - - - - - - - - - - - - - -
    Type conversion functionsDescription
    - {% highlight java %} -ANY.cast(TYPE) -{% endhighlight %} - -

    Converts a value to a given type. E.g. "42".cast(INT) leads to 42.

    -
    - - - - - - - - - - - - - - - - - -
    Value constructor functionsDescription
    - {% highlight java %} -NUMERIC.rows -{% endhighlight %} - -

    Creates an interval of rows.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Temporal functionsDescription
    - {% highlight java %} -STRING.toDate() -{% endhighlight %} - -

    Parses a date string in the form "yy-mm-dd" to a SQL date.

    -
    - {% highlight java %} -STRING.toTime() -{% endhighlight %} - -

    Parses a time string in the form "hh:mm:ss" to a SQL time.

    -
    - {% highlight java %} -STRING.toTimestamp() -{% endhighlight %} - -

    Parses a timestamp string in the form "yy-mm-dd hh:mm:ss.fff" to a SQL timestamp.

    -
    - {% highlight java %} -NUMERIC.year -NUMERIC.years -{% endhighlight %} - -

    Creates an interval of months for a given number of years.

    -
    - {% highlight java %} -NUMERIC.month -NUMERIC.months -{% endhighlight %} - -

    Creates an interval of months for a given number of months.

    -
    - {% highlight java %} -NUMERIC.day -NUMERIC.days -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of days.

    -
    - {% highlight java %} -NUMERIC.hour -NUMERIC.hours -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of hours.

    -
    - {% highlight java %} -NUMERIC.minute -NUMERIC.minutes -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of minutes.

    -
    - {% highlight java %} -NUMERIC.second -NUMERIC.seconds -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of seconds.

    -
    - {% highlight java %} -NUMERIC.milli -NUMERIC.millis -{% endhighlight %} - -

    Creates an interval of milliseconds.

    -
    - {% highlight java %} -currentDate() -{% endhighlight %} - -

    Returns the current SQL date in UTC time zone.

    -
    - {% highlight java %} -currentTime() -{% endhighlight %} - -

    Returns the current SQL time in UTC time zone.

    -
    - {% highlight java %} -currentTimestamp() -{% endhighlight %} - -

    Returns the current SQL timestamp in UTC time zone.

    -
    - {% highlight java %} -localTime() -{% endhighlight %} - -

    Returns the current SQL time in local time zone.

    -
    - {% highlight java %} -localTimestamp() -{% endhighlight %} - -

    Returns the current SQL timestamp in local time zone.

    -
    - {% highlight java %} -TEMPORAL.extract(TIMEINTERVALUNIT) -{% endhighlight %} - -

    Extracts parts of a time point or time interval. Returns the part as a long value. E.g. '2006-06-05'.toDate.extract(DAY) leads to 5.

    -
    - {% highlight java %} -TIMEPOINT.floor(TIMEINTERVALUNIT) -{% endhighlight %} - -

    Rounds a time point down to the given unit. E.g. '12:44:31'.toDate.floor(MINUTE) leads to 12:44:00.

    -
    - {% highlight java %} -TIMEPOINT.ceil(TIMEINTERVALUNIT) -{% endhighlight %} - -

    Rounds a time point up to the given unit. E.g. '12:44:31'.toTime.floor(MINUTE) leads to 12:45:00.

    -
    - {% highlight java %} -DATE.quarter() -{% endhighlight %} - -

    Returns the quarter of a year from a SQL date. E.g. '1994-09-27'.toDate.quarter() leads to 3.

    -
    - {% highlight java %} -temporalOverlaps(TIMEPOINT, TEMPORAL, TIMEPOINT, TEMPORAL) -{% endhighlight %} - -

    Determines whether two anchored time intervals overlap. Time point and temporal are transformed into a range defined by two time points (start, end). The function evaluates leftEnd >= rightStart && rightEnd >= leftStart. E.g. temporalOverlaps("2:55:00".toTime, 1.hour, "3:30:00".toTime, 2.hour) leads to true.

    -
    - {% highlight java %} -dateFormat(TIMESTAMP, STRING) -{% endhighlight %} - -

    Formats timestamp as a string using a specified format. The format must be compatible with MySQL's date formatting syntax as used by the date_parse function. The format specification is given in the Date Format Specifier table below.

    -

    For example dateFormat(ts, '%Y, %d %M') results in strings formatted as "2017, 05 May".

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Aggregate functionsDescription
    - {% highlight java %} -FIELD.count -{% endhighlight %} - -

    Returns the number of input rows for which the field is not null.

    -
    - {% highlight java %} -FIELD.avg -{% endhighlight %} - -

    Returns the average (arithmetic mean) of the numeric field across all input values.

    -
    - {% highlight java %} -FIELD.sum -{% endhighlight %} - -

    Returns the sum of the numeric field across all input values. If all values are null, null is returned.

    -
    - {% highlight java %} -FIELD.sum0 -{% endhighlight %} - -

    Returns the sum of the numeric field across all input values. If all values are null, 0 is returned.

    -
    - {% highlight java %} -FIELD.max -{% endhighlight %} - -

    Returns the maximum value of field across all input values.

    -
    - {% highlight java %} -FIELD.min -{% endhighlight %} - -

    Returns the minimum value of field across all input values.

    -
    - {% highlight java %} -FIELD.stddevPop -{% endhighlight %} - -

    Returns the population standard deviation of the numeric field across all input values.

    -
    - {% highlight java %} -FIELD.stddevSamp -{% endhighlight %} - -

    Returns the sample standard deviation of the numeric field across all input values.

    -
    - {% highlight java %} -FIELD.varPop -{% endhighlight %} - -

    Returns the population variance (square of the population standard deviation) of the numeric field across all input values.

    -
    - {% highlight java %} -FIELD.varSamp -{% endhighlight %} - -

    Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.

    -
    - {% highlight java %} -FIELD.collect - {% endhighlight %} - -

    Returns the multiset aggregate of the input value.

    -
    - - - - - - - - - - - - - - - - - - - - - - -
    Value access functionsDescription
    - {% highlight java %} -COMPOSITE.get(STRING) -COMPOSITE.get(INT) -{% endhighlight %} - -

    Accesses the field of a Flink composite type (such as Tuple, POJO, etc.) by index or name and returns it's value. E.g. pojo.get('myField') or tuple.get(0).

    -
    - {% highlight java %} -ANY.flatten() -{% endhighlight %} - -

    Converts a Flink composite type (such as Tuple, POJO, etc.) and all of its direct subtypes into a flat representation where every subtype is a separate field. In most cases the fields of the flat representation are named similarly to the original fields but with a dollar separator (e.g. mypojo$mytuple$f0).

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Array functionsDescription
    - {% highlight java %} -array(ANY [, ANY ]*) -{% endhighlight %} - -

    Creates an array from a list of values. The array will be an array of objects (not primitives).

    -
    - {% highlight java %} -ARRAY.cardinality() -{% endhighlight %} - -

    Returns the number of elements of an array.

    -
    - {% highlight java %} -ARRAY.at(INT) -{% endhighlight %} - -

    Returns the element at a particular position in an array. The index starts at 1.

    -
    - {% highlight java %} -ARRAY.element() -{% endhighlight %} - -

    Returns the sole element of an array with a single element. Returns null if the array is empty. Throws an exception if the array has more than one element.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Map functionsDescription
    - {% highlight java %} -map(ANY, ANY [, ANY, ANY ]*) -{% endhighlight %} - -

    Creates a map from a list of key-value pairs.

    -
    - {% highlight java %} -MAP.cardinality() -{% endhighlight %} - -

    Returns the number of entries of a map.

    -
    - {% highlight java %} -MAP.at(ANY) -{% endhighlight %} - -

    Returns the value specified by a particular key in a map.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Hash functionsDescription
    - {% highlight java %} -STRING.md5() -{% endhighlight %} - -

    Returns the MD5 hash of the string argument as a string of 32 hexadecimal digits; null if string is null.

    -
    - {% highlight java %} -STRING.sha1() -{% endhighlight %} - -

    Returns the SHA-1 hash of the string argument as a string of 40 hexadecimal digits; null if string is null.

    -
    - {% highlight java %} -STRING.sha256() -{% endhighlight %} - -

    Returns the SHA-256 hash of the string argument as a string of 64 hexadecimal digits; null if string is null.

    -
    - - - - - - - - - - - - - - - - - -
    Row functionsDescription
    - {% highlight java %} -row(ANY, [, ANY]*) -{% endhighlight %} - -

    Creates a row from a list of values. Row is composite type and can be access via value access functions.

    -
    - - - - - - - - - - - - - - - - - -
    Auxiliary functionsDescription
    - {% highlight java %} -ANY.as(name [, name ]* ) -{% endhighlight %} - -

    Specifies a name for an expression i.e. a field. Additional names can be specified if the expression expands to multiple fields.

    -
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Comparison functionsDescription
    - {% highlight scala %} -ANY === ANY -{% endhighlight %} - -

    Equals.

    -
    - {% highlight scala %} -ANY !== ANY -{% endhighlight %} - -

    Not equal.

    -
    - {% highlight scala %} -ANY > ANY -{% endhighlight %} - -

    Greater than.

    -
    - {% highlight scala %} -ANY >= ANY -{% endhighlight %} - -

    Greater than or equal.

    -
    - {% highlight scala %} -ANY < ANY -{% endhighlight %} - -

    Less than.

    -
    - {% highlight scala %} -ANY <= ANY -{% endhighlight %} - -

    Less than or equal.

    -
    - {% highlight scala %} -ANY.isNull -{% endhighlight %} - -

    Returns true if the given expression is null.

    -
    - {% highlight scala %} -ANY.isNotNull -{% endhighlight %} - -

    Returns true if the given expression is not null.

    -
    - {% highlight scala %} -STRING.like(STRING) -{% endhighlight %} - -

    Returns true, if a string matches the specified LIKE pattern. E.g. "Jo_n%" matches all strings that start with "Jo(arbitrary letter)n".

    -
    - {% highlight scala %} -STRING.similar(STRING) -{% endhighlight %} - -

    Returns true, if a string matches the specified SQL regex pattern. E.g. "A+" matches all strings that consist of at least one "A".

    -
    - {% highlight scala %} -ANY.in(ANY, ANY, ...) -{% endhighlight %} - -

    Returns true if an expression exists in a given list of expressions. This is a shorthand for multiple OR conditions. If the testing set contains null, the result will be null if the element can not be found and true if it can be found. If element is null, the result is always null. E.g. "42".in(1, 2, 3) leads to false.

    -
    - {% highlight scala %} -ANY.in(TABLE) -{% endhighlight %} - -

    Returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression. Note: This operation is not supported in a streaming environment yet.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Logical functionsDescription
    - {% highlight scala %} -boolean1 || boolean2 -{% endhighlight %} - -

    Returns true if boolean1 is true or boolean2 is true. Supports three-valued logic.

    -
    - {% highlight scala %} -boolean1 && boolean2 -{% endhighlight %} - -

    Returns true if boolean1 and boolean2 are both true. Supports three-valued logic.

    -
    - {% highlight scala %} -!BOOLEAN -{% endhighlight %} - -

    Returns true if boolean expression is not true; returns null if boolean is null.

    -
    - {% highlight scala %} -BOOLEAN.isTrue -{% endhighlight %} - -

    Returns true if the given boolean expression is true. False otherwise (for null and false).

    -
    - {% highlight scala %} -BOOLEAN.isFalse -{% endhighlight %} - -

    Returns true if given boolean expression is false. False otherwise (for null and true).

    -
    - {% highlight scala %} -BOOLEAN.isNotTrue -{% endhighlight %} - -

    Returns true if the given boolean expression is not true (for null and false). False otherwise.

    -
    - {% highlight scala %} -BOOLEAN.isNotFalse -{% endhighlight %} - -

    Returns true if given boolean expression is not false (for null and true). False otherwise.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Arithmetic functionsDescription
    - {% highlight scala %} -+ numeric -{% endhighlight %} - -

    Returns numeric.

    -
    - {% highlight scala %} -- numeric -{% endhighlight %} - -

    Returns negative numeric.

    -
    - {% highlight scala %} -numeric1 + numeric2 -{% endhighlight %} - -

    Returns numeric1 plus numeric2.

    -
    - {% highlight scala %} -numeric1 - numeric2 -{% endhighlight %} - -

    Returns numeric1 minus numeric2.

    -
    - {% highlight scala %} -numeric1 * numeric2 -{% endhighlight %} - -

    Returns numeric1 multiplied by numeric2.

    -
    - {% highlight scala %} -numeric1 / numeric2 -{% endhighlight %} - -

    Returns numeric1 divided by numeric2.

    -
    - {% highlight scala %} -numeric1.power(numeric2) -{% endhighlight %} - -

    Returns numeric1 raised to the power of numeric2.

    -
    - {% highlight scala %} -NUMERIC.abs() -{% endhighlight %} - -

    Calculates the absolute value of given value.

    -
    - {% highlight scala %} -numeric1 % numeric2 -{% endhighlight %} - -

    Returns the remainder (modulus) of numeric1 divided by numeric2. The result is negative only if numeric1 is negative.

    -
    - {% highlight scala %} -NUMERIC.sqrt() -{% endhighlight %} - -

    Calculates the square root of a given value.

    -
    - {% highlight scala %} -NUMERIC.ln() -{% endhighlight %} - -

    Calculates the natural logarithm of given value.

    -
    - {% highlight scala %} -NUMERIC.log10() -{% endhighlight %} - -

    Calculates the base 10 logarithm of given value.

    -
    - {% highlight scala %} -NUMERIC.exp() -{% endhighlight %} - -

    Calculates the Euler's number raised to the given power.

    -
    - {% highlight scala %} -NUMERIC.ceil() -{% endhighlight %} - -

    Calculates the smallest integer greater than or equal to a given number.

    -
    - {% highlight scala %} -NUMERIC.floor() -{% endhighlight %} - -

    Calculates the largest integer less than or equal to a given number.

    -
    - {% highlight scala %} -NUMERIC.sin() -{% endhighlight %} - -

    Calculates the sine of a given number.

    -
    - {% highlight scala %} -NUMERIC.cos() -{% endhighlight %} - -

    Calculates the cosine of a given number.

    -
    - {% highlight scala %} -NUMERIC.tan() -{% endhighlight %} - -

    Calculates the cotangent of a given number.

    -
    - {% highlight scala %} -NUMERIC.cot() -{% endhighlight %} - -

    Calculates the arc sine of a given number.

    -
    - {% highlight scala %} -NUMERIC.asin() -{% endhighlight %} - -

    Calculates the arc cosine of a given number.

    -
    - {% highlight scala %} -NUMERIC.acos() -{% endhighlight %} - -

    Calculates the arc tangent of a given number.

    -
    - {% highlight scala %} -NUMERIC.atan() -{% endhighlight %} - -

    Calculates the tangent of a given number.

    -
    - {% highlight scala %} -NUMERIC.degrees() -{% endhighlight %} - -

    Converts numeric from radians to degrees.

    -
    - {% highlight scala %} -NUMERIC.radians() -{% endhighlight %} - -

    Converts numeric from degrees to radians.

    -
    - {% highlight scala %} -NUMERIC.sign() -{% endhighlight %} - -

    Calculates the signum of a given number.

    -
    - {% highlight scala %} -NUMERIC.round(INT) -{% endhighlight %} - -

    Rounds the given number to integer places right to the decimal point.

    -
    - {% highlight scala %} -pi() -{% endhighlight %} - -

    Returns a value that is closer than any other value to pi.

    -
    - {% highlight scala %} -e() -{% endhighlight %} - -

    Returns a value that is closer than any other value to e.

    -
    - {% highlight scala %} -rand() -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive).

    -
    - {% highlight scala %} -rand(seed integer) -{% endhighlight %} - -

    Returns a pseudorandom double value between 0.0 (inclusive) and 1.0 (exclusive) with a initial seed. Two rand functions will return identical sequences of numbers if they have same initial seed.

    -
    - {% highlight scala %} -randInteger(bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive).

    -
    - {% highlight scala %} -randInteger(seed integer, bound integer) -{% endhighlight %} - -

    Returns a pseudorandom integer value between 0.0 (inclusive) and the specified value (exclusive) with a initial seed. Two randInteger functions will return identical sequences of numbers if they have same initial seed and same bound.

    -
    - {% highlight scala %} -NUMERIC.bin() -{% endhighlight %} - -

    Returns a string representation of an integer numeric value in binary format. Returns null if numeric is null. E.g. "4" leads to "100", "12" leads to "1100".

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Arithmetic functionsDescription
    - {% highlight scala %} -STRING + STRING -{% endhighlight %} - -

    Concatenates two character strings.

    -
    - {% highlight scala %} -STRING.charLength() -{% endhighlight %} - -

    Returns the length of a String.

    -
    - {% highlight scala %} -STRING.upperCase() -{% endhighlight %} - -

    Returns all of the characters in a string in upper case using the rules of the default locale.

    -
    - {% highlight scala %} -STRING.lowerCase() -{% endhighlight %} - -

    Returns all of the characters in a string in lower case using the rules of the default locale.

    -
    - {% highlight scala %} -STRING.position(STRING) -{% endhighlight %} - -

    Returns the position of string in an other string starting at 1. Returns 0 if string could not be found. E.g. "a".position("bbbbba") leads to 6.

    -
    - {% highlight scala %} -STRING.trim( - leading = true, - trailing = true, - character = " ") -{% endhighlight %} - -

    Removes leading and/or trailing characters from the given string.

    -
    - {% highlight scala %} -STRING.overlay(STRING, INT) -STRING.overlay(STRING, INT, INT) -{% endhighlight %} - -

    Replaces a substring of string with a string starting at a position (starting at 1). An optional length specifies how many characters should be removed. E.g. "xxxxxtest".overlay("xxxx", 6) leads to "xxxxxxxxx", "xxxxxtest".overlay('xxxx', 6, 2) leads to "xxxxxxxxxst".

    -
    - {% highlight scala %} -STRING.substring(INT) -{% endhighlight %} - -

    Creates a substring of the given string beginning at the given index to the end. The start index starts at 1 and is inclusive.

    -
    - {% highlight scala %} -STRING.substring(INT, INT) -{% endhighlight %} - -

    Creates a substring of the given string at the given index for the given length. The index starts at 1 and is inclusive, i.e., the character at the index is included in the substring. The substring has the specified length or less.

    -
    - {% highlight scala %} -STRING.initCap() -{% endhighlight %} - -

    Converts the initial letter of each word in a string to uppercase. Assumes a string containing only [A-Za-z0-9], everything else is treated as whitespace.

    -
    - - - - - - - - - - - - - - - - - -
    Conditional functionsDescription
    - {% highlight java %} -BOOLEAN.?(value1, value2) -{% endhighlight %} - -

    Ternary conditional operator that decides which of two other expressions should be evaluated based on a evaluated boolean condition. E.g. (42 > 5).?("A", "B") leads to "A".

    -
    - - - - - - - - - - - - - - - - - -
    Type conversion functionsDescription
    - {% highlight scala %} -ANY.cast(TYPE) -{% endhighlight %} - -

    Converts a value to a given type. E.g. "42".cast(Types.INT) leads to 42.

    -
    - - - - - - - - - - - - - - - - - -
    Value constructor functionsDescription
    - {% highlight scala %} -NUMERIC.rows -{% endhighlight %} - -

    Creates an interval of rows.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Temporal functionsDescription
    - {% highlight scala %} -STRING.toDate -{% endhighlight %} - -

    Parses a date string in the form "yy-mm-dd" to a SQL date.

    -
    - {% highlight scala %} -STRING.toTime -{% endhighlight %} - -

    Parses a time string in the form "hh:mm:ss" to a SQL time.

    -
    - {% highlight scala %} -STRING.toTimestamp -{% endhighlight %} - -

    Parses a timestamp string in the form "yy-mm-dd hh:mm:ss.fff" to a SQL timestamp.

    -
    - {% highlight scala %} -NUMERIC.year -NUMERIC.years -{% endhighlight %} - -

    Creates an interval of months for a given number of years.

    -
    - {% highlight scala %} -NUMERIC.month -NUMERIC.months -{% endhighlight %} - -

    Creates an interval of months for a given number of months.

    -
    - {% highlight scala %} -NUMERIC.day -NUMERIC.days -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of days.

    -
    - {% highlight scala %} -NUMERIC.hour -NUMERIC.hours -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of hours.

    -
    - {% highlight scala %} -NUMERIC.minute -NUMERIC.minutes -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of minutes.

    -
    - {% highlight scala %} -NUMERIC.second -NUMERIC.seconds -{% endhighlight %} - -

    Creates an interval of milliseconds for a given number of seconds.

    -
    - {% highlight scala %} -NUMERIC.milli -NUMERIC.millis -{% endhighlight %} - -

    Creates an interval of milliseconds.

    -
    - {% highlight scala %} -currentDate() -{% endhighlight %} - -

    Returns the current SQL date in UTC time zone.

    -
    - {% highlight scala %} -currentTime() -{% endhighlight %} - -

    Returns the current SQL time in UTC time zone.

    -
    - {% highlight scala %} -currentTimestamp() -{% endhighlight %} - -

    Returns the current SQL timestamp in UTC time zone.

    -
    - {% highlight scala %} -localTime() -{% endhighlight %} - -

    Returns the current SQL time in local time zone.

    -
    - {% highlight scala %} -localTimestamp() -{% endhighlight %} - -

    Returns the current SQL timestamp in local time zone.

    -
    - {% highlight scala %} -TEMPORAL.extract(TimeIntervalUnit) -{% endhighlight %} - -

    Extracts parts of a time point or time interval. Returns the part as a long value. E.g. "2006-06-05".toDate.extract(TimeIntervalUnit.DAY) leads to 5.

    -
    - {% highlight scala %} -TIMEPOINT.floor(TimeIntervalUnit) -{% endhighlight %} - -

    Rounds a time point down to the given unit. E.g. "12:44:31".toTime.floor(TimeIntervalUnit.MINUTE) leads to 12:44:00.

    -
    - {% highlight scala %} -TIMEPOINT.ceil(TimeIntervalUnit) -{% endhighlight %} - -

    Rounds a time point up to the given unit. E.g. "12:44:31".toTime.floor(TimeIntervalUnit.MINUTE) leads to 12:45:00.

    -
    - {% highlight scala %} -DATE.quarter() -{% endhighlight %} - -

    Returns the quarter of a year from a SQL date. E.g. "1994-09-27".toDate.quarter() leads to 3.

    -
    - {% highlight scala %} -temporalOverlaps(TIMEPOINT, TEMPORAL, TIMEPOINT, TEMPORAL) -{% endhighlight %} - -

    Determines whether two anchored time intervals overlap. Time point and temporal are transformed into a range defined by two time points (start, end). The function evaluates leftEnd >= rightStart && rightEnd >= leftStart. E.g. temporalOverlaps('2:55:00'.toTime, 1.hour, '3:30:00'.toTime, 2.hours) leads to true.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Aggregate functionsDescription
    - {% highlight scala %} -FIELD.count -{% endhighlight %} - -

    Returns the number of input rows for which the field is not null.

    -
    - {% highlight scala %} -FIELD.avg -{% endhighlight %} - -

    Returns the average (arithmetic mean) of the numeric field across all input values.

    -
    - {% highlight scala %} -FIELD.sum -{% endhighlight %} - -

    Returns the sum of the numeric field across all input values. If all values are null, null is returned.

    -
    - {% highlight scala %} -FIELD.sum0 -{% endhighlight %} - -

    Returns the sum of the numeric field across all input values. If all values are null, 0 is returned.

    -
    - {% highlight scala %} -FIELD.max -{% endhighlight %} - -

    Returns the maximum value of field across all input values.

    -
    - {% highlight scala %} -FIELD.min -{% endhighlight %} - -

    Returns the minimum value of field across all input values.

    -
    - {% highlight scala %} -FIELD.stddevPop -{% endhighlight %} - -

    Returns the population standard deviation of the numeric field across all input values.

    -
    - {% highlight scala %} -FIELD.stddevSamp -{% endhighlight %} - -

    Returns the sample standard deviation of the numeric field across all input values.

    -
    - {% highlight scala %} -FIELD.varPop -{% endhighlight %} - -

    Returns the population variance (square of the population standard deviation) of the numeric field across all input values.

    -
    - {% highlight scala %} -FIELD.varSamp -{% endhighlight %} - -

    Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.

    -
    - {% highlight scala %} -FIELD.collect - {% endhighlight %} - -

    Returns the multiset aggregate of the input value.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Value access functionsDescription
    - {% highlight scala %} -COMPOSITE.get(STRING) -COMPOSITE.get(INT) -{% endhighlight %} - -

    Accesses the field of a Flink composite type (such as Tuple, POJO, etc.) by index or name and returns it's value. E.g. 'pojo.get("myField") or 'tuple.get(0).

    -
    - {% highlight scala %} -ANY.flatten() -{% endhighlight %} - -

    Converts a Flink composite type (such as Tuple, POJO, etc.) and all of its direct subtypes into a flat representation where every subtype is a separate field. In most cases the fields of the flat representation are named similarly to the original fields but with a dollar separator (e.g. mypojo$mytuple$f0).

    -
    - {% highlight scala %} -dateFormat(TIMESTAMP, STRING) -{% endhighlight %} - -

    Formats timestamp as a string using a specified format. The format must be compatible with MySQL's date formatting syntax as used by the date_parse function. The format specification is given in the Date Format Specifier table below.

    -

    For example dateFormat('ts, "%Y, %d %M") results in strings formatted as "2017, 05 May".

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Array functionsDescription
    - {% highlight scala %} -array(ANY [, ANY ]*) -{% endhighlight %} - -

    Creates an array from a list of values. The array will be an array of objects (not primitives).

    -
    - {% highlight scala %} -ARRAY.cardinality() -{% endhighlight %} - -

    Returns the number of elements of an array.

    -
    - {% highlight scala %} -ARRAY.at(INT) -{% endhighlight %} - -

    Returns the element at a particular position in an array. The index starts at 1.

    -
    - {% highlight scala %} -ARRAY.element() -{% endhighlight %} - -

    Returns the sole element of an array with a single element. Returns null if the array is empty. Throws an exception if the array has more than one element.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Map functionsDescription
    - {% highlight scala %} -map(ANY, ANY [, ANY, ANY ]*) -{% endhighlight %} - -

    Creates a map from a list of key-value pairs.

    -
    - {% highlight scala %} -MAP.cardinality() -{% endhighlight %} - -

    Returns the number of entries of a map.

    -
    - {% highlight scala %} -MAP.at(ANY) -{% endhighlight %} - -

    Returns the value specified by a particular key in a map.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Hash functionsDescription
    - {% highlight scala %} -STRING.md5() -{% endhighlight %} - -

    Returns the MD5 hash of the string argument as a string of 32 hexadecimal digits; null if string is null.

    -
    - {% highlight scala %} -STRING.sha1() -{% endhighlight %} - -

    Returns the SHA-1 hash of the string argument as a string of 40 hexadecimal digits; null if string is null.

    -
    - {% highlight scala %} -STRING.sha256() -{% endhighlight %} - -

    Returns the SHA-256 hash of the string argument as a string of 64 hexadecimal digits; null if string is null.

    -
    - - - - - - - - - - - - - - - - - -
    Row functionsDescription
    - {% highlight scala %} -row(ANY, [, ANY]*) -{% endhighlight %} - -

    Creates a row from a list of values. Row is composite type and can be access via value access functions.

    -
    - - - - - - - - - - - - - - - - -
    Auxiliary functionsDescription
    - {% highlight scala %} -ANY.as(name [, name ]* ) -{% endhighlight %} - -

    Specifies a name for an expression i.e. a field. Additional names can be specified if the expression expands to multiple fields.

    -
    -
    - -
    - -### Unsupported Functions - -The following operations are not supported yet: - -- Binary string operators and functions -- System functions -- Aggregate functions like REGR_xxx -- Distinct aggregate functions like COUNT DISTINCT - -{% top %} diff --git a/docs/dev/table/udfs.md b/docs/dev/table/udfs.md index 02c047e67db272..d93b5c8c3f7972 100644 --- a/docs/dev/table/udfs.md +++ b/docs/dev/table/udfs.md @@ -72,7 +72,7 @@ tableEnv.registerFunction("hashCode", new HashCode(10)); myTable.select("string, string.hashCode(), hashCode(string)"); // use the function in SQL API -tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable"); +tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable"); {% endhighlight %}
    @@ -93,7 +93,7 @@ myTable.select('string, hashCode('string)) // register and use the function in SQL tableEnv.registerFunction("hashCode", new HashCode(10)) -tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable") +tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable") {% endhighlight %}
    @@ -110,8 +110,8 @@ public static class TimestampModifier extends ScalarFunction { return t % 1000; } - public TypeInformation getResultType(signature: Class[]) { - return Types.TIMESTAMP; + public TypeInformation getResultType(Class[] signature) { + return Types.SQL_TIMESTAMP; } } {% endhighlight %} @@ -141,7 +141,7 @@ Similar to a user-defined scalar function, a user-defined table function takes z In order to define a table function one has to extend the base class `TableFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a table function is determined by its evaluation methods. An evaluation method must be declared `public` and named `eval`. The `TableFunction` can be overloaded by implementing multiple methods named `eval`. The parameter types of the evaluation methods determine all valid parameters of the table function. Evaluation methods can also support variable arguments, such as `eval(String... strs)`. The type of the returned table is determined by the generic type of `TableFunction`. Evaluation methods emit output rows using the protected `collect(T)` method. -In the Table API, a table function is used with `.join(Expression)` or `.leftOuterJoin(Expression)` for Scala users and `.join(String)` or `.leftOuterJoin(String)` for Java users. The `join` operator (cross) joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator). The `leftOuterJoin` operator joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator) and preserves outer rows for which the table function returns an empty table. In SQL use `LATERAL TABLE()` with CROSS JOIN and LEFT JOIN with an ON TRUE join condition (see examples below). +In the Table API, a table function is used with `.join(Table)` or `.leftOuterJoin(Table)`. The `join` operator (cross) joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator). The `leftOuterJoin` operator joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator) and preserves outer rows for which the table function returns an empty table. In SQL use `LATERAL TABLE()` with CROSS JOIN and LEFT JOIN with an ON TRUE join condition (see examples below). The following example shows how to define table-valued function, register it in the TableEnvironment, and call it in a query. Note that you can configure your table function via a constructor before it is registered: @@ -171,8 +171,10 @@ Table myTable = ... // table schema: [a: String] tableEnv.registerFunction("split", new Split("#")); // Use the table function in the Java Table API. "as" specifies the field names of the table. -myTable.join("split(a) as (word, length)").select("a, word, length"); -myTable.leftOuterJoin("split(a) as (word, length)").select("a, word, length"); +myTable.join(new Table(tableEnv, "split(a) as (word, length)")) + .select("a, word, length"); +myTable.leftOuterJoin(new Table(tableEnv, "split(a) as (word, length)")) + .select("a, word, length"); // Use the table function in SQL with LATERAL and TABLE keywords. // CROSS JOIN a table function (equivalent to "join" in Table API). @@ -188,7 +190,7 @@ tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(s class Split(separator: String) extends TableFunction[(String, Int)] { def eval(str: String): Unit = { // use collect(...) to emit a row. - str.split(separator).foreach(x -> collect((x, x.length)) + str.split(separator).foreach(x => collect((x, x.length))) } } @@ -208,7 +210,7 @@ tableEnv.registerFunction("split", new Split("#")) // CROSS JOIN a table function (equivalent to "join" in Table API) tableEnv.sqlQuery("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) as T(word, length)") // LEFT JOIN a table function (equivalent to "leftOuterJoin" in Table API) -tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN TABLE(split(a)) as T(word, length) ON TRUE") +tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE") {% endhighlight %} **IMPORTANT:** Do not implement TableFunction as a Scala object. Scala object is a singleton and will cause concurrency issues.
    @@ -228,7 +230,7 @@ public class CustomTypeSplit extends TableFunction { for (String s : str.split(" ")) { Row row = new Row(2); row.setField(0, s); - row.setField(1, s.length); + row.setField(1, s.length()); collect(row); } } @@ -292,7 +294,7 @@ optionally implemented. While some of these methods allow the system more effici **The following methods of `AggregateFunction` are required depending on the use case:** - `retract()` is required for aggregations on bounded `OVER` windows. -- `merge()` is required for many batch aggreagtions and session window aggregations. +- `merge()` is required for many batch aggregations and session window aggregations. - `resetAccumulator()` is required for many batch aggregations. All methods of `AggregateFunction` must be declared as `public`, not `static` and named exactly as the names mentioned above. The methods `createAccumulator`, `getValue`, `getResultType`, and `getAccumulatorType` are defined in the `AggregateFunction` abstract class, while others are contracted methods. In order to define a aggregate function, one has to extend the base class `org.apache.flink.table.functions.AggregateFunction` and implement one (or more) `accumulate` methods. The method `accumulate` can be overloaded with different parameter types and supports variable arguments. @@ -464,7 +466,7 @@ abstract class AggregateFunction[T, ACC] extends UserDefinedFunction { */ def getValue(accumulator: ACC): T // MANDATORY - h/** + /** * Resets the accumulator for this [[AggregateFunction]]. This function must be implemented for * dataset grouping aggregate. * diff --git a/docs/dev/types_serialization.md b/docs/dev/types_serialization.md index a0f497b391199b..9bace8cdbeef46 100644 --- a/docs/dev/types_serialization.md +++ b/docs/dev/types_serialization.md @@ -238,7 +238,7 @@ as possible via reflection, using the few bits that Java preserves (mainly funct This logic also contains some simple type inference for cases where the return type of a function depends on its input type: {% highlight java %} -public class AppendOne extends MapFunction> { +public class AppendOne implements MapFunction> { public Tuple2 map(T value) { return new Tuple2(value, 1L); diff --git a/docs/docker/Dockerfile b/docs/docker/Dockerfile index ad4bb307f7c6af..b34235e2a36435 100644 --- a/docs/docker/Dockerfile +++ b/docs/docker/Dockerfile @@ -14,8 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM centos:centos7 - -RUN yum install -y vim gem ruby-devel make gcc gcc-c++ python-setuptools && \ - gem install bundler +FROM fedora:28 +RUN set -ex; \ + dnf install -y \ + vim \ + gem \ + ruby-devel \ + redhat-rpm-config \ + make \ + gcc \ + gcc-c++ \ + python-setuptools \ + ; \ + gem install bundler; \ + dnf clean all diff --git a/docs/examples/index.md b/docs/examples/index.md index 38d9dd679a4e58..f7242241167e77 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -25,15 +25,26 @@ specific language governing permissions and limitations under the License. --> -[Sample Project in Java]({{ site.baseurl }}/quickstart/java_api_quickstart.html) and [Sample Project in Scala]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) are guides to setting up Maven and SBT projects and include simple implementations of a word count application. -[Monitoring Wikipedia Edits]({{ site.baseurl }}/quickstart/run_example_quickstart.html) is a more complete example of a streaming analytics application. +## Bundled Examples -[Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana. +The Flink sources include many examples for Flink's different APIs: -The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. See the hands-on sections, and the exercises. +* DataStream applications ({% gh_link flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples "Java" %} / {% gh_link flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples "Scala" %}) +* DataSet applications ({% gh_link flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala "Scala" %}) +* Table API / SQL queries ({% gh_link flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala "Scala" %}) -## Bundled Examples +These [instructions]({{ site.baseurl }}/dev/batch/examples.html#running-an-example) explain how to run the examples. + +## Examples on the Web + +There are also a few blog posts published online that discuss example applications: + +* [How to build stateful streaming applications with Apache Flink +](https://www.infoworld.com/article/3293426/big-data/how-to-build-stateful-streaming-applications-with-apache-flink.html) presents an event-driven application implemented with the DataStream API and two SQL queries for streaming analytics. + +* [Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana. -The Flink sources include a number of examples for both **streaming** ( [java](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples) / [scala](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples) ) and **batch** ( [java](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java) / [scala](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala) ). These [instructions]({{ site.baseurl }}/dev/batch/examples.html#running-an-example) explain how to run the examples. +* The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. Check out the hands-on sections and the exercises. +{% top %} diff --git a/docs/fig/interval-join.svg b/docs/fig/interval-join.svg new file mode 100644 index 00000000000000..25b59324548589 --- /dev/null +++ b/docs/fig/interval-join.svg @@ -0,0 +1,147 @@ + + + + + + Interval Join (Selected) + Created with Sketch. + + + + + time + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 6 + + + + + + 7 + + + + + + 3 + + + + + + 4 + + + + + + 5 + + + + + + 7 + + + + + + 0 + + + + 0,0 + 0,1 + + + 2,0 + 2,1 + + + 3,1 + + + 5,6 + + + 7,6 + 7,7 + + + + + + + + + + + 1 + + + + lower bound + + + upper bound + + + + + 2 + + + + + + 0 + + + + + diff --git a/docs/fig/local_recovery.png b/docs/fig/local_recovery.png new file mode 100644 index 00000000000000..01c429535cf077 Binary files /dev/null and b/docs/fig/local_recovery.png differ diff --git a/docs/fig/session-window-join.svg b/docs/fig/session-window-join.svg new file mode 100644 index 00000000000000..8fb3293edc98ec --- /dev/null +++ b/docs/fig/session-window-join.svg @@ -0,0 +1,125 @@ + + + + + + Session Window Join + Created with Sketch. + + + + + + + + + + time + + + + + 2 + + + + + + 5 + + + + + + 6 + + + + + + 6 + + + + + + 8 + + + + + + 9 + + + + + + 1 + + + + + + 5 + + + + + + 4 + + + + + + 0 + + + + + + + + + + + + + + + GAP + + + GAP + + + 1,0 + 2,0 + + + 5,4 + 5,5 + 6,4 + 6,5 + + + + diff --git a/docs/fig/sliding-window-join.svg b/docs/fig/sliding-window-join.svg new file mode 100644 index 00000000000000..f544e0de35ff26 --- /dev/null +++ b/docs/fig/sliding-window-join.svg @@ -0,0 +1,113 @@ + + + + + + Sliding Window Join + Created with Sketch. + + + + + + + time + + + + + + + + + + + + 2 + + + + + + 3 + + + + + + 4 + + + + + + 1 + + + + + + 3 + + + + + + 4 + + + + + + 0 + + + + + + 0 + + + + 0,0 + + + 0,0 + 1,0 + + + 2,3 + 3,3 + + + 3,3 + 3,4 + 4,3 + 4,4 + + + 4,4 + + + - + + + + diff --git a/docs/fig/sql_client_demo.gif b/docs/fig/sql_client_demo.gif new file mode 100644 index 00000000000000..bb0554e4d0e04e Binary files /dev/null and b/docs/fig/sql_client_demo.gif differ diff --git a/docs/fig/ssl_internal_external.svg b/docs/fig/ssl_internal_external.svg new file mode 100755 index 00000000000000..04262d29cbc272 --- /dev/null +++ b/docs/fig/ssl_internal_external.svg @@ -0,0 +1,336 @@ + + + + + + + + + image/svg+xml + + + + + + + + + Task + Manager + + Task + Manager + + Task + Manager + + Resource + Manager + + Job + Manager + + Master Process + + + + + + + + + + + RPC / BLOB + Data Plane + Data Plane + + + + REST + Internal + External + + Browser + + CLI + + Tools + + + HTTP + + Dispatcher + + + diff --git a/docs/fig/tumbling-window-join.svg b/docs/fig/tumbling-window-join.svg new file mode 100644 index 00000000000000..9efb58c3173350 --- /dev/null +++ b/docs/fig/tumbling-window-join.svg @@ -0,0 +1,131 @@ + + + + + + Tumbling Window Join + Created with Sketch. + + + + + time + + + + + + + 2 + + + + + + 3 + + + + + + 4 + + + + + + 5 + + + + + + 6 + + + + + + 7 + + + + + + 0 + + + + + + 1 + + + + + + + + 3 + + + + + + 4 + + + + + + 0 + + + + + + 1 + + + + + + + + + + 0,0 + 0,1 + 1,0 + 1,1 + + + 2,3 + 3,3 + + + 4,4 + 5,4 + + + + + + diff --git a/docs/flinkDev/building.md b/docs/flinkDev/building.md new file mode 100644 index 00000000000000..034215b1ff37ad --- /dev/null +++ b/docs/flinkDev/building.md @@ -0,0 +1,134 @@ +--- +title: Building Flink from Source +nav-parent_id: flinkdev +nav-pos: 20 +--- + + +This page covers how to build Flink {{ site.version }} from sources. + +* This will be replaced by the TOC +{:toc} + +## Build Flink + +In order to build Flink you need the source code. Either [download the source of a release]({{ site.download_url }}) or [clone the git repository]({{ site.github_url }}). + +In addition you need **Maven 3** and a **JDK** (Java Development Kit). Flink requires **at least Java 8** to build. + +*NOTE: Maven 3.3.x can build Flink, but will not properly shade away certain dependencies. Maven 3.2.5 creates the libraries properly. +To build unit tests use Java 8u51 or above to prevent failures in unit tests that use the PowerMock runner.* + +To clone from git, enter: + +{% highlight bash %} +git clone {{ site.github_url }} +{% endhighlight %} + +The simplest way of building Flink is by running: + +{% highlight bash %} +mvn clean install -DskipTests +{% endhighlight %} + +This instructs [Maven](http://maven.apache.org) (`mvn`) to first remove all existing builds (`clean`) and then create a new Flink binary (`install`). + +To speed up the build you can skip tests, QA plugins, and JavaDocs: + +{% highlight bash %} +mvn clean install -DskipTests -Dfast +{% endhighlight %} + +The default build adds a Flink-specific JAR for Hadoop 2, to allow using Flink with HDFS and YARN. + +## Dependency Shading + +Flink [shades away](https://maven.apache.org/plugins/maven-shade-plugin/) some of the libraries it uses, in order to avoid version clashes with user programs that use different versions of these libraries. Among the shaded libraries are *Google Guava*, *Asm*, *Apache Curator*, *Apache HTTP Components*, *Netty*, and others. + +The dependency shading mechanism was recently changed in Maven and requires users to build Flink slightly differently, depending on their Maven version: + +**Maven 3.0.x, 3.1.x, and 3.2.x** +It is sufficient to call `mvn clean install -DskipTests` in the root directory of Flink code base. + +**Maven 3.3.x** +The build has to be done in two steps: First in the base directory, then in the distribution project: + +{% highlight bash %} +mvn clean install -DskipTests +cd flink-dist +mvn clean install +{% endhighlight %} + +*Note:* To check your Maven version, run `mvn --version`. + +{% top %} + +## Hadoop Versions + +{% info %} Most users do not need to do this manually. The [download page]({{ site.download_url }}) contains binary packages for common Hadoop versions. + +Flink has dependencies to HDFS and YARN which are both dependencies from [Apache Hadoop](http://hadoop.apache.org). There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If you are using a wrong combination of versions, exceptions can occur. + +Hadoop is only supported from version 2.4.0 upwards. +You can also specify a specific Hadoop version to build against: + +{% highlight bash %} +mvn clean install -DskipTests -Dhadoop.version=2.6.1 +{% endhighlight %} + +### Vendor-specific Versions + +To build Flink against a vendor specific Hadoop version, issue the following command: + +{% highlight bash %} +mvn clean install -DskipTests -Pvendor-repos -Dhadoop.version=2.6.1-cdh5.0.0 +{% endhighlight %} + +The `-Pvendor-repos` activates a Maven [build profile](http://maven.apache.org/guides/introduction/introduction-to-profiles.html) that includes the repositories of popular Hadoop vendors such as Cloudera, Hortonworks, or MapR. + +{% top %} + +## Scala Versions + +{% info %} Users that purely use the Java APIs and libraries can *ignore* this section. + +Flink has APIs, libraries, and runtime modules written in [Scala](http://scala-lang.org). Users of the Scala API and libraries may have to match the Scala version of Flink with the Scala version of their projects (because Scala is not strictly backwards compatible). + +Since version 1.7 Flink builds with Scala version 2.11 and 2.12. + +{% top %} + +## Encrypted File Systems + +If your home directory is encrypted you might encounter a `java.io.IOException: File name too long` exception. Some encrypted file systems, like encfs used by Ubuntu, do not allow long filenames, which is the cause of this error. + +The workaround is to add: + +{% highlight xml %} + + -Xmax-classfile-name + 128 + +{% endhighlight %} + +in the compiler configuration of the `pom.xml` file of the module causing the error. For example, if the error appears in the `flink-yarn` module, the above code should be added under the `` tag of `scala-maven-plugin`. See [this issue](https://issues.apache.org/jira/browse/FLINK-2003) for more information. + +{% top %} + diff --git a/docs/flinkDev/ide_setup.md b/docs/flinkDev/ide_setup.md new file mode 100644 index 00000000000000..87e7ceb5e9936c --- /dev/null +++ b/docs/flinkDev/ide_setup.md @@ -0,0 +1,127 @@ +--- +title: "Importing Flink into an IDE" +nav-parent_id: flinkdev +nav-pos: 3 +--- + + +* Replaced by the TOC +{:toc} + +The sections below describe how to import the Flink project into an IDE +for the development of Flink itself. For writing Flink programs, please +refer to the [Java API]({{ site.baseurl }}/dev/projectsetup/java_api_quickstart.html) +and the [Scala API]({{ site.baseurl }}/dev/projectsetup/scala_api_quickstart.html) +quickstart guides. + +**NOTE:** Whenever something is not working in your IDE, try with the Maven +command line first (`mvn clean package -DskipTests`) as it might be your IDE +that has a bug or is not properly set up. + +## Preparation + +To get started, please first checkout the Flink sources from one of our +[repositories](https://flink.apache.org/community.html#source-code), +e.g. +{% highlight bash %} +git clone https://github.com/apache/flink.git +{% endhighlight %} + +## IntelliJ IDEA + +A brief guide on how to set up IntelliJ IDEA IDE for development of the Flink core. +As Eclipse is known to have issues with mixed Scala and Java projects, more and more contributors are migrating to IntelliJ IDEA. + +The following documentation describes the steps to setup IntelliJ IDEA 2016.2.5 +([https://www.jetbrains.com/idea/download/](https://www.jetbrains.com/idea/download/)) +with the Flink sources. + +### Installing the Scala plugin + +The IntelliJ installation setup offers to install the Scala plugin. +If it is not installed, follow these instructions before importing Flink +to enable support for Scala projects and files: + +1. Go to IntelliJ plugins settings (IntelliJ IDEA -> Preferences -> Plugins) and + click on "Install Jetbrains plugin...". +2. Select and install the "Scala" plugin. +3. Restart IntelliJ + +### Importing Flink + +1. Start IntelliJ IDEA and choose "Import Project" +2. Select the root folder of the Flink repository +3. Choose "Import project from external model" and select "Maven" +4. Leave the default options and click on "Next" until you hit the SDK section. +5. If there is no SDK, create a one with the "+" sign top left, + then click "JDK", select your JDK home directory and click "OK". + Otherwise simply select your SDK. +6. Continue by clicking "Next" again and finish the import. +7. Right-click on the imported Flink project -> Maven -> Generate Sources and Update Folders. + Note that this will install Flink libraries in your local Maven repository, + i.e. "/home/*-your-user-*/.m2/repository/org/apache/flink/". + Alternatively, `mvn clean package -DskipTests` also creates the necessary + files for the IDE to work with but without installing libraries. +8. Build the Project (Build -> Make Project) + +### Checkstyle For Java +IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin. + +1. Install the "Checkstyle-IDEA" plugin from the IntelliJ plugin repository. +2. Configure the plugin by going to Settings -> Other Settings -> Checkstyle. +3. Set the "Scan Scope" to "Only Java sources (including tests)". +4. Select _8.9_ in the "Checkstyle Version" dropdown and click apply. **This step is important, + don't skip it!** +5. In the "Configuration File" pane, add a new configuration using the plus icon: + 1. Set the "Description" to "Flink". + 2. Select "Use a local Checkstyle file", and point it to + `"tools/maven/checkstyle.xml"` within + your repository. + 3. Check the box for "Store relative to project location", and click + "Next". + 4. Configure the "checkstyle.suppressions.file" property value to + `"suppressions.xml"`, and click "Next", then "Finish". +6. Select "Flink" as the only active configuration file, and click "Apply" and + "OK". +7. Checkstyle will now give warnings in the editor for any Checkstyle + violations. + +Once the plugin is installed you can directly import `"tools/maven/checkstyle.xml"` by going to Settings -> Editor -> Code Style -> Java -> Gear Icon next to Scheme dropbox. This will for example automatically adjust the imports layout. + +You can scan an entire module by opening the Checkstyle tools window and +clicking the "Check Module" button. The scan should report no errors. + +Note Some modules are not fully covered by checkstyle, +which include `flink-core`, `flink-optimizer`, and `flink-runtime`. +Nevertheless please make sure that code you add/modify in these modules still conforms to the checkstyle rules. + +### Checkstyle For Scala + +Enable scalastyle in Intellij by selecting Settings -> Editor -> Inspections, then searching for "Scala style inspections". Also Place `"tools/maven/scalastyle_config.xml"` in the `"/.idea"` or `"/project"` directory. + +## Eclipse + +**NOTE:** From our experience, this setup does not work with Flink +due to deficiencies of the old Eclipse version bundled with Scala IDE 3.0.3 or +due to version incompatibilities with the bundled Scala version in Scala IDE 4.4.1. + +**We recommend to use IntelliJ instead (see [above](#intellij-idea))** + +{% top %} diff --git a/docs/flinkDev/index.md b/docs/flinkDev/index.md new file mode 100644 index 00000000000000..462feae47620b9 --- /dev/null +++ b/docs/flinkDev/index.md @@ -0,0 +1,26 @@ +--- +section-break: true +nav-title: ' Flink Development' +title: "Flink Development" +nav-id: "flinkdev" +nav-parent_id: root +nav-pos: 8 +--- + diff --git a/docs/index.md b/docs/index.md index 1ba8f07c10043d..f6ae8a2f910805 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,9 +31,11 @@ Apache Flink is an open source platform for distributed stream and batch data pr ## First Steps -- **Concepts**: Start with the basic concepts of Flink's [Dataflow Programming Model](concepts/programming-model.html) and [Distributed Runtime Environment](concepts/runtime.html). This will help you understand other parts of the documentation, including the setup and programming guides. We recommended you read these sections first. +- **Concepts**: Start with the basic concepts of Flink's [Dataflow Programming Model](concepts/programming-model.html) and [Distributed Runtime Environment](concepts/runtime.html). This will help you understand other parts of the documentation, including the setup and programming guides. We recommend you read these sections first. -- **Quickstarts**: [Run an example program](quickstart/setup_quickstart.html) on your local machine or [study some examples](examples/index.html). +- **Tutorials**: + * [Implement and run a DataStream application](./tutorials/datastream_api.html) + * [Setup a local Flink cluster](./tutorials/local_setup.html) - **Programming Guides**: You can read our guides about [basic API concepts](dev/api_concepts.html) and the [DataStream API](dev/datastream_api.html) or the [DataSet API](dev/batch/index.html) to learn how to write your first Flink programs. @@ -41,12 +43,14 @@ Apache Flink is an open source platform for distributed stream and batch data pr Before putting your Flink job into production, read the [Production Readiness Checklist](ops/production_ready.html). -## Migration Guide +## Release Notes -For users of earlier versions of Apache Flink, we recommend the [API migration guide](dev/migration.html). -While all parts of the API that were marked as public and stable are still supported (the public API is backwards compatible), we suggest migrating applications to the newer interfaces where applicable. +Release notes cover important changes between Flink versions. Please carefully read these notes if you plan to upgrade your Flink setup to a later version. -For users that plan to upgrade a Flink system in production, we recommend reading the guide on [upgrading Apache Flink](ops/upgrading.html). +* [Release notes for Flink 1.8](release-notes/flink-1.8.html). +* [Release notes for Flink 1.7](release-notes/flink-1.7.html). +* [Release notes for Flink 1.6](release-notes/flink-1.6.html). +* [Release notes for Flink 1.5](release-notes/flink-1.5.html). ## External Resources diff --git a/docs/internals/components.md b/docs/internals/components.md index e85183b92a1c61..bb949f20a80ee2 100644 --- a/docs/internals/components.md +++ b/docs/internals/components.md @@ -53,7 +53,7 @@ You can click on the components in the figure to learn more. - + diff --git a/docs/internals/ide_setup.md b/docs/internals/ide_setup.md deleted file mode 100644 index 340544d7346702..00000000000000 --- a/docs/internals/ide_setup.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: "IDE Setup" -nav-parent_id: start -nav-pos: 3 ---- - - -* Replaced by the TOC -{:toc} - -The sections below describe how to import the Flink project into an IDE -for the development of Flink itself. For writing Flink programs, please -refer to the [Java API]({{ site.baseurl }}/quickstart/java_api_quickstart.html) -and the [Scala API]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) -quickstart guides. - -**NOTE:** Whenever something is not working in your IDE, try with the Maven -command line first (`mvn clean package -DskipTests`) as it might be your IDE -that has a bug or is not properly set up. - -## Preparation - -To get started, please first checkout the Flink sources from one of our -[repositories](https://flink.apache.org/community.html#source-code), -e.g. -{% highlight bash %} -git clone https://github.com/apache/flink.git -{% endhighlight %} - -## IntelliJ IDEA - -A brief guide on how to set up IntelliJ IDEA IDE for development of the Flink core. -As Eclipse is known to have issues with mixed Scala and Java projects, more and more contributors are migrating to IntelliJ IDEA. - -The following documentation describes the steps to setup IntelliJ IDEA 2016.2.5 -([https://www.jetbrains.com/idea/download/](https://www.jetbrains.com/idea/download/)) -with the Flink sources. - -### Installing the Scala plugin - -The IntelliJ installation setup offers to install the Scala plugin. -If it is not installed, follow these instructions before importing Flink -to enable support for Scala projects and files: - -1. Go to IntelliJ plugins settings (IntelliJ IDEA -> Preferences -> Plugins) and - click on "Install Jetbrains plugin...". -2. Select and install the "Scala" plugin. -3. Restart IntelliJ - -### Importing Flink - -1. Start IntelliJ IDEA and choose "Import Project" -2. Select the root folder of the Flink repository -3. Choose "Import project from external model" and select "Maven" -4. Leave the default options and click on "Next" until you hit the SDK section. -5. If there is no SDK, create a one with the "+" sign top left, - then click "JDK", select your JDK home directory and click "OK". - Otherwise simply select your SDK. -6. Continue by clicking "Next" again and finish the import. -7. Right-click on the imported Flink project -> Maven -> Generate Sources and Update Folders. - Note that this will install Flink libraries in your local Maven repository, - i.e. "/home/*-your-user-*/.m2/repository/org/apache/flink/". - Alternatively, `mvn clean package -DskipTests` also creates the necessary - files for the IDE to work with but without installing libraries. -8. Build the Project (Build -> Make Project) - -### Checkstyle -IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin. - -1. Install the "Checkstyle-IDEA" plugin from the IntelliJ plugin repository. -2. Configure the plugin by going to Settings -> Other Settings -> Checkstyle. -3. Set the "Scan Scope" to "Only Java sources (including tests)". -4. Select _8.4_ in the "Checkstyle Version" dropdown and click apply. **This step is important, - don't skip it!** -5. In the "Configuration File" pane, add a new configuration using the plus icon: - 1. Set the "Description" to "Flink". - 2. Select "Use a local Checkstyle file", and point it to - `"tools/maven/checkstyle.xml"` within - your repository. - 3. Check the box for "Store relative to project location", and click - "Next". - 4. Configure the "checkstyle.suppressions.file" property value to - `"suppressions.xml"`, and click "Next", then "Finish". -6. Select "Flink" as the only active configuration file, and click "Apply" and - "OK". -7. Checkstyle will now give warnings in the editor for any Checkstyle - violations. - -Once the plugin is installed you can directly import `"tools/maven/checkstyle.xml"` by going to Settings -> Editor -> Code Style -> Java -> Gear Icon next to Scheme dropbox. This will for example automatically adjust the imports layout. - -You can scan an entire module by opening the Checkstyle tools window and -clicking the "Check Module" button. The scan should report no errors. - -Note Some modules are not fully covered by checkstyle, -which include `flink-core`, `flink-optimizer`, and `flink-runtime`. -Nevertheless please make sure that code you add/modify in these modules still conforms to the checkstyle rules. - -## Eclipse - -**NOTE:** From our experience, this setup does not work with Flink -due to deficiencies of the old Eclipse version bundled with Scala IDE 3.0.3 or -due to version incompatibilities with the bundled Scala version in Scala IDE 4.4.1. - -**We recommend to use IntelliJ instead (see [above](#intellij-idea))** - -{% top %} diff --git a/docs/internals/index.md b/docs/internals/index.md index 8b3184cc945069..bd079a98dc4788 100644 --- a/docs/internals/index.md +++ b/docs/internals/index.md @@ -1,10 +1,9 @@ --- title: "Internals" nav-id: internals -nav-pos: 8 +nav-pos: 9 nav-title: ' Internals' nav-parent_id: root -section-break: true --- + +**Docker / Kubernetes Sessions** + +Docker / Kubernetes setups that start first a set of JobManagers / TaskManagers and then submit jobs/applications via REST or the CLI +behave like standalone sessions: Flink's code is in the Java classpath, the job's code is loaded dynamically. -When starting a Flink cluster, the JobManagers and TaskManagers are started with the Flink framework classes in the -classpath. The classes from all jobs that are submitted against the cluster are loaded *dynamically*. **YARN** YARN classloading differs between single job deployments and sessions: - - When submitting a Flink job directly to YARN (via `bin/flink run -m yarn-cluster ...`), dedicated TaskManagers and - JobManagers are started for that job. Those JVMs have both Flink framework classes and user code classes in their classpath. + - When submitting a Flink job/application directly to YARN (via `bin/flink run -m yarn-cluster ...`), dedicated TaskManagers and + JobManagers are started for that job. Those JVMs have both Flink framework classes and user code classes in the Java classpath. That means that there is *no dynamic classloading* involved in that case. - When starting a YARN session, the JobManagers and TaskManagers are started with the Flink framework classes in the @@ -58,20 +76,35 @@ YARN classloading differs between single job deployments and sessions: **Mesos** Mesos setups following [this documentation](../ops/deployment/mesos.html) currently behave very much like the a -YARN session: The TaskManager and JobManager processes are started with the Flink framework classes in classpath, job +YARN session: The TaskManager and JobManager processes are started with the Flink framework classes in the Java classpath, job classes are loaded dynamically when the jobs are submitted. -## Configuring ClassLoader Resolution Order -Flink uses a hierarchy of ClassLoaders for loading classes from the user-code jar(s). The user-code -ClassLoader has a reference to the parent ClassLoader, which is the default Java ClassLoader in most -cases. By default, Java ClassLoaders will first look for classes in the parent ClassLoader and then in -the child ClassLoader for cases where we have a hierarchy of ClassLoaders. This is problematic if you -have in your user jar a version of a library that conflicts with a version that comes with Flink. You can -change this behaviour by configuring the ClassLoader resolution order via -`classloader.resolve-order: child-first` in the Flink config. However, Flink classes will still -be resolved through the parent ClassLoader first, although you can also configure this via -`classloader.parent-first-patterns` (see [config](../ops/config.html)) +## Inverted Class Loading and ClassLoader Resolution Order + +In setups where dynamic classloading is involved (sessions), there is a hierarchy of typically two ClassLoaders: +(1) Java's *application classloader*, which has all classes in the classpath, and (2) the dynamic *user code classloader*. +for loading classes from the user-code jar(s). The user-code ClassLoader has the application classloader as its parent. + +By default, Flink inverts classloading order, meaning it looks into the user code classloader first, and only looks into +the parent (application classloader) if the class is not part of the dynamically loaded user code. + +The benefit of inverted classloading is that jobs can use different library versions than Flink's core itself, which is very +useful when the different versions of the libraries are not compatible. The mechanism helps to avoid the common dependency conflict +errors like `IllegalAccessError` or `NoSuchMethodError`. Different parts of the code simply have separate copies of the +classes (Flink's core or one of its dependencies can use a different copy than the user code). +In most cases, this work well and no additional configuration from the user is needed. + +However, there are cases when the inverted classloading causes problems (see below, "X cannot be cast to X"). +You can revert back to Java's default mode by configuring the ClassLoader resolution order via +[classloader.resolve-order](../ops/config.html#classloader-resolve-order) in the Flink config to `parent-first` +(from Flink's default `child-first`). + +Please note that certain classes are always resolved in a *parent-first* way (through the parent ClassLoader first), because they +are shared between Flink's core and the user code or the user-code facing APIs. The packages for these classes are configured via +[classloader.parent-first-patterns-default](../ops/config.html#classloader-parent-first-patterns-default) and +[classloader.parent-first-patterns-additional](../ops/config.html#classloader-parent-first-patterns-additional). +To add new packages to be *parent-first* loaded, please set the `classloader.parent-first-patterns-additional` config option. ## Avoiding Dynamic Classloading @@ -103,26 +136,22 @@ and access the user code class loader via `getRuntimeContext().getUserCodeClassL ## X cannot be cast to X exceptions -When you see an exception in the style `com.foo.X cannot be cast to com.foo.X`, it means that multiple versions of the class -`com.foo.X` have been loaded by different class loaders, and types of that class are attempted to be assigned to each other. - -The reason is in most cases that an object of the `com.foo.X` class loaded from a previous execution attempt is still cached somewhere, -and picked up by a restarted task/operator that reloaded the code. Note that this is again only possible in deployments that use -dynamic class loading. +In setups with dynamic classloading, you may see an exception in the style `com.foo.X cannot be cast to com.foo.X`. +This means that multiple versions of the class `com.foo.X` have been loaded by different class loaders, and types of that class are attempted to be assigned to each other. -Common causes of cached object instances: +One common reason is that a library is not compatible with Flink's *inverted classloading* approach. You can turn off inverted classloading +to verify this (set [`classloader.resolve-order: parent-first`](../ops/config.html#classloader-resolve-order) in the Flink config) or exclude +the library from inverted classloading (set [`classloader.parent-first-patterns-additional`](../ops/config.html#classloader-parent-first-patterns-additional) +in the Flink config). - - When using *Apache Avro*: The *SpecificDatumReader* caches instances of records. Avoid using `SpecificData.INSTANCE`. See also - [this discussion](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/How-to-get-help-on-ClassCastException-when-re-submitting-a-job-tp10972p11133.html) - - - Using certain serialization frameworks for cloning objects (such as *Apache Avro*) - - - Interning objects (for example via Guava's Interners) +Another cause can be cached object instances, as produced by some libraries like *Apache Avro*, or by interning objects (for example via Guava's Interners). +The solution here is to either have a setup without any dynamic classloading, or to make sure that the respective library is fully part of the dynamically loaded code. +The latter means that the library must not be added to Flink's `/lib` folder, but must be part of the application's fat-jar/uber-jar ## Unloading of Dynamically Loaded Classes -All scenarios that involve dynamic class loading (i.e., standalone, sessions, mesos, ...) rely on classes being *unloaded* again. +All scenarios that involve dynamic class loading (sessions) rely on classes being *unloaded* again. Class unloading means that the Garbage Collector finds that no objects from a class exist and more, and thus removes the class (the code, static variable, metadata, etc). @@ -141,15 +170,14 @@ Common causes for class leaks and suggested fixes: ## Resolving Dependency Conflicts with Flink using the maven-shade-plugin. -Apache Flink loads many classes by default into its classpath. If a user uses a different version of a library that Flink is using, often `IllegalAccessExceptions` or `NoSuchMethodError` are the result. +A way to address dependency conflicts from the application developer's side is to avoid exposing dependencies by *shading them away*. -Through Hadoop, Flink for example depends on the `aws-sdk` library or on `protobuf-java`. If your user code is using these libraries and you run into issues we recommend relocating the dependency in your user code jar. - -Apache Maven offers the [maven-shade-plugin](https://maven.apache.org/plugins/maven-shade-plugin/), which allows one to change the package of a class *after* compiling it (so the code you are writing is not affected by the shading). For example if you have the `com.amazonaws` packages from the aws sdk in your user code jar, the shade plugin would relocate them into the `org.myorg.shaded.com.amazonaws` package, so that your code is calling your aws sdk version. +Apache Maven offers the [maven-shade-plugin](https://maven.apache.org/plugins/maven-shade-plugin/), which allows one to change the package of a +class *after* compiling it (so the code you are writing is not affected by the shading). For example if you have the `com.amazonaws` packages from +the aws sdk in your user code jar, the shade plugin would relocate them into the `org.myorg.shaded.com.amazonaws` package, so that your code is calling your aws sdk version. This documentation page explains [relocating classes using the shade plugin](https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html). - -Note that some of Flink's dependencies, such as `guava` are shaded away by the maintainers of Flink, so users usually don't have to worry about it. +Note that most of Flink's dependencies, such as `guava`, `netty`, `jackson`, etc. are shaded away by the maintainers of Flink, so users usually don't have to worry about it. {% top %} diff --git a/docs/monitoring/debugging_event_time.md b/docs/monitoring/debugging_event_time.md index 10a3fb20fdb267..8ac72ebe6632f3 100644 --- a/docs/monitoring/debugging_event_time.md +++ b/docs/monitoring/debugging_event_time.md @@ -33,13 +33,13 @@ is tracked within the system. Low watermarks of each task can be accessed through Flink web interface or [metrics system]({{ site.baseurl }}/monitoring/metrics.html). -Each Task in Flink exposes a metric called `currentLowWatermark` that represents the lowest watermark received +Each Task in Flink exposes a metric called `currentInputWatermark` that represents the lowest watermark received by this task. This long value represents the "current event time". The value is calculated by taking the minimum of all watermarks received by upstream operators. This means that the event time tracked with watermarks is always dominated by the furthest-behind source. The low watermark metric is accessible **using the web interface**, by choosing a task in the metric tab, -and selecting the `.currentLowWatermark` metric. In the new box you'll now be able to see +and selecting the `.currentInputWatermark` metric. In the new box you'll now be able to see the current low watermark of the task. Another way of getting the metric is using one of the **metric reporters**, as described in the documentation diff --git a/docs/monitoring/historyserver.md b/docs/monitoring/historyserver.md index dfbbc8edbc1d4c..7f8f737e2200c6 100644 --- a/docs/monitoring/historyserver.md +++ b/docs/monitoring/historyserver.md @@ -35,10 +35,10 @@ The HistoryServer allows you to query the status and statistics of completed job After you have configured the HistoryServer *and* JobManager, you start and stop the HistoryServer via its corresponding startup script: -```sh +{% highlight shell %} # Start or stop the HistoryServer -bin/historyserver.sh (start|stop) -``` +bin/historyserver.sh (start|start-foreground|stop) +{% endhighlight %} By default, this server binds to `localhost` and listens at port `8082`. @@ -52,22 +52,22 @@ The configuration keys `jobmanager.archive.fs.dir` and `historyserver.archive.fs The archiving of completed jobs happens on the JobManager, which uploads the archived job information to a file system directory. You can configure the directory to archive completed jobs in `flink-conf.yaml` by setting a directory via `jobmanager.archive.fs.dir`. -```sh +{% highlight yaml %} # Directory to upload completed job information jobmanager.archive.fs.dir: hdfs:///completed-jobs -``` +{% endhighlight %} **HistoryServer** The HistoryServer can be configured to monitor a comma-separated list of directories in via `historyserver.archive.fs.dir`. The configured directories are regularly polled for new archives; the polling interval can be configured via `historyserver.archive.fs.refresh-interval`. -```sh +{% highlight yaml %} # Monitor the following directories for completed jobs historyserver.archive.fs.dir: hdfs:///completed-jobs # Refresh every 10 seconds historyserver.archive.fs.refresh-interval: 10000 -``` +{% endhighlight %} The contained archives are downloaded and cached in the local filesystem. The local directory for this is configured via `historyserver.web.tmpdir`. diff --git a/docs/monitoring/logging.md b/docs/monitoring/logging.md index b548d416693ee5..737ddb9de3031c 100644 --- a/docs/monitoring/logging.md +++ b/docs/monitoring/logging.md @@ -46,7 +46,7 @@ The configuration file either has to be specified by setting the environment pro The `conf` directory contains a `logback.xml` file which can be modified and is used if Flink is started outside of an IDE and with the provided starting scripts. The provided `logback.xml` has the following form: -~~~ xml +{% highlight xml %} ${log.file} @@ -60,13 +60,13 @@ The provided `logback.xml` has the following form: -~~~ +{% endhighlight %} In order to control the logging level of `org.apache.flink.runtime.jobgraph.JobGraph`, for example, one would have to add the following line to the configuration file. -~~~ xml +{% highlight xml %} -~~~ +{% endhighlight %} For further information on configuring logback see [LOGback's manual](http://logback.qos.ch/manual/configuration.html). @@ -74,27 +74,27 @@ For further information on configuring logback see [LOGback's manual](http://log The loggers using slf4j are created by calling -~~~ java +{% highlight java %} import org.slf4j.LoggerFactory import org.slf4j.Logger Logger LOG = LoggerFactory.getLogger(Foobar.class) -~~~ +{% endhighlight %} In order to benefit most from slf4j, it is recommended to use its placeholder mechanism. Using placeholders allows to avoid unnecessary string constructions in case that the logging level is set so high that the message would not be logged. The syntax of placeholders is the following: -~~~ java +{% highlight java %} LOG.info("This message contains {} placeholders. {}", 2, "Yippie"); -~~~ +{% endhighlight %} Placeholders can also be used in conjunction with exceptions which shall be logged. -~~~ java +{% highlight java %} catch(Exception exception){ LOG.error("An {} occurred.", "error", exception); } -~~~ +{% endhighlight %} {% top %} diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index 62adeb184a543f..ae51f9ee3d4422 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -182,7 +182,7 @@ new class MyMapper extends RichMapFunction[String,String] { override def open(parameters: Configuration): Unit = { getRuntimeContext() .getMetricGroup() - .gauge("MyGauge", ScalaGauge[Int]( () => valueToExpose ) ) + .gauge[Int, ScalaGauge[Int]]("MyGauge", ScalaGauge[Int]( () => valueToExpose ) ) } override def map(value: String): String = { @@ -426,7 +426,7 @@ class MyMapper extends RichMapFunction[Long,Long] { Every metric is assigned an identifier and a set of key-value pairs under which the metric will be reported. -THe identifier is based on 3 components: the user-defined name when registering the metric, an optional user-defined scope and a system-provided scope. +The identifier is based on 3 components: a user-defined name when registering the metric, an optional user-defined scope and a system-provided scope. For example, if `A.B` is the system scope, `C.D` the user scope and `E` the name, then the identifier for the metric will be `A.B.C.D.E`. You can configure which delimiter to use for the identifier (default: `.`) by setting the `metrics.scope.delimiter` key in `conf/flink-conf.yaml`. @@ -569,7 +569,7 @@ we will list more settings specific to each reporter. Example reporter configuration that specifies multiple reporters: -``` +{% highlight yaml %} metrics.reporters: my_jmx_reporter,my_other_reporter metrics.reporter.my_jmx_reporter.class: org.apache.flink.metrics.jmx.JMXReporter @@ -579,7 +579,7 @@ metrics.reporter.my_other_reporter.class: org.apache.flink.metrics.graphite.Grap metrics.reporter.my_other_reporter.host: 192.168.1.1 metrics.reporter.my_other_reporter.port: 10000 -``` +{% endhighlight %} **Important:** The jar containing the reporter must be accessible when Flink is started by placing it in the /lib folder. @@ -595,10 +595,11 @@ but not activated. Parameters: -- `port` - (optional) the port on which JMX listens for connections. This can also be a port range. When a -range is specified the actual port is shown in the relevant job or task manager log. If this setting is set -Flink will start an extra JMX connector for the given port/range. Metrics are always available on the default -local JMX interface. +- `port` - (optional) the port on which JMX listens for connections. +In order to be able to run several instances of the reporter on one host (e.g. when one TaskManager is colocated with the JobManager) it is advisable to use a port range like `9250-9260`. +When a range is specified the actual port is shown in the relevant job or task manager log. +If this setting is set Flink will start an extra JMX connector for the given port/range. +Metrics are always available on the default local JMX interface. Example configuration: @@ -621,34 +622,6 @@ An example for such a list would be `host=localhost,job_name=MyJob,task_name=MyT The domain thus identifies a metric class, while the key-property list identifies one (or multiple) instances of that metric. -### Ganglia (org.apache.flink.metrics.ganglia.GangliaReporter) - -In order to use this reporter you must copy `/opt/flink-metrics-ganglia-{{site.version}}.jar` into the `/lib` folder -of your Flink distribution. - -Parameters: - -- `host` - the gmond host address configured under `udp_recv_channel.bind` in `gmond.conf` -- `port` - the gmond port configured under `udp_recv_channel.port` in `gmond.conf` -- `tmax` - soft limit for how long an old metric should be retained -- `dmax` - hard limit for how long an old metric should be retained -- `ttl` - time-to-live for transmitted UDP packets -- `addressingMode` - UDP addressing mode to use (UNICAST/MULTICAST) - -Example configuration: - -{% highlight yaml %} - -metrics.reporter.gang.class: org.apache.flink.metrics.ganglia.GangliaReporter -metrics.reporter.gang.host: localhost -metrics.reporter.gang.port: 8649 -metrics.reporter.gang.tmax: 60 -metrics.reporter.gang.dmax: 0 -metrics.reporter.gang.ttl: 1 -metrics.reporter.gang.addressingMode: MULTICAST - -{% endhighlight %} - ### Graphite (org.apache.flink.metrics.graphite.GraphiteReporter) In order to use this reporter you must copy `/opt/flink-metrics-graphite-{{site.version}}.jar` into the `/lib` folder @@ -673,12 +646,13 @@ metrics.reporter.grph.protocol: TCP ### Prometheus (org.apache.flink.metrics.prometheus.PrometheusReporter) -In order to use this reporter you must copy `/opt/flink-metrics-prometheus-{{site.version}}.jar` into the `/lib` folder +In order to use this reporter you must copy `/opt/flink-metrics-prometheus{{site.scala_version_suffix}}-{{site.version}}.jar` into the `/lib` folder of your Flink distribution. Parameters: - `port` - (optional) the port the Prometheus exporter listens on, defaults to [9249](https://github.com/prometheus/prometheus/wiki/Default-port-allocations). In order to be able to run several instances of the reporter on one host (e.g. when one TaskManager is colocated with the JobManager) it is advisable to use a port range like `9250-9260`. +- `filterLabelValueCharacters` - (optional) Specifies whether to filter label value characters. If enabled, all characters not matching \[a-zA-Z0-9:_\] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the [Prometheus requirements](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). Example configuration: @@ -699,6 +673,32 @@ Flink metric types are mapped to Prometheus metric types as follows: All Flink metrics variables (see [List of all Variables](#list-of-all-variables)) are exported to Prometheus as labels. +### PrometheusPushGateway (org.apache.flink.metrics.prometheus.PrometheusPushGatewayReporter) + +In order to use this reporter you must copy `/opt/flink-metrics-prometheus-{{site.version}}.jar` into the `/lib` folder +of your Flink distribution. + +Parameters: + +{% include generated/prometheus_push_gateway_reporter_configuration.html %} + +Example configuration: + +{% highlight yaml %} + +metrics.reporter.promgateway.class: org.apache.flink.metrics.prometheus.PrometheusPushGatewayReporter +metrics.reporter.promgateway.host: localhost +metrics.reporter.promgateway.port: 9091 +metrics.reporter.promgateway.jobName: myJob +metrics.reporter.promgateway.randomJobNameSuffix: true +metrics.reporter.promgateway.deleteOnShutdown: false + +{% endhighlight %} + +The PrometheusPushGatewayReporter pushes metrics to a [Pushgateway](https://github.com/prometheus/pushgateway), which can be scraped by Prometheus. + +Please see the [Prometheus documentation](https://prometheus.io/docs/practices/pushing/) for use-cases. + ### StatsD (org.apache.flink.metrics.statsd.StatsDReporter) In order to use this reporter you must copy `/opt/flink-metrics-statsd-{{site.version}}.jar` into the `/lib` folder @@ -1106,7 +1106,7 @@ Thus, in order to infer the metric identifier: fullRestarts - The total number of full restarts since this job was submitted (in milliseconds). + The total number of full restarts since this job was submitted. Gauge @@ -1178,6 +1178,9 @@ Thus, in order to infer the metric identifier: +### RocksDB +Certain RocksDB native metrics are available but disabled by default, you can find full documentation [here]({{ site.baseurl }}/ops/config.html#rocksdb-native-metrics) + ### IO @@ -1190,7 +1193,13 @@ Thus, in order to infer the metric identifier: - + + + + + + + @@ -1210,6 +1219,26 @@ Thus, in order to infer the metric identifier: + + + + + + + + + + + + + + + + + + + + @@ -1221,7 +1250,17 @@ Thus, in order to infer the metric identifier: - + + + + + + + + + + + @@ -1247,15 +1286,15 @@ Thus, in order to infer the metric identifier: - + - - - - - @@ -1365,13 +1399,224 @@ Thus, in order to infer the metric identifier: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TaskJob (only available on TaskManager)<source_id>.<source_subtask_index>.<operator_id>.<operator_subtask_index>.latencyThe latency distributions from a given source subtask to an operator subtask (in milliseconds).Histogram
    Task numBytesInLocal The total number of bytes this task has read from a local source. CounterThe number of bytes this task reads from a remote source per second. Meter
    numBuffersInLocalThe total number of network buffers this task has read from a local source.Counter
    numBuffersInLocalPerSecondThe number of network buffers this task reads from a local source per second.Meter
    numBuffersInRemoteThe total number of network buffers this task has read from a remote source.Counter
    numBuffersInRemotePerSecondThe number of network buffers this task reads from a remote source per second.Meter
    numBytesOut The total number of bytes this task has emitted.Meter
    Task/OperatornumBuffersOutThe total number of network buffers this task has emitted.Counter
    numBuffersOutPerSecondThe number of network buffers this task emits per second.Meter
    Task/Operator numRecordsIn The total number of records this operator/task has received. CounterCounter
    Operator currentInputWatermark - The last watermark this operator has received (in milliseconds). -

    Note: For operators with 2 inputs this is the minimum of the last received watermarks.

    + The last watermark this operator/tasks has received (in milliseconds). +

    Note: For operators/tasks with 2 inputs this is the minimum of the last received watermarks.

    		 Gauge
    Operator currentInput1Watermark The last watermark this operator has received in its first input (in milliseconds). @@ -1278,11 +1317,6 @@ Thus, in order to infer the metric identifier: Gauge
    latencyThe latency distributions from all incoming sources (in milliseconds).Histogram
    numSplitsProcessed The total number of InputSplits this data source has processed (if the operator is a data source). Gauge
    OperatorsleepTimeMillisstream, shardIdThe number of milliseconds the consumer spends sleeping before fetching records from Kinesis. + A particular shard's metric can be specified by stream name and shard id. + Gauge
    OperatormaxNumberOfRecordsPerFetchstream, shardIdThe maximum number of records requested by the consumer in a single getRecords call to Kinesis. If ConsumerConfigConstants.SHARD_USE_ADAPTIVE_READS + is set to true, this value is adaptively calculated to maximize the 2 Mbps read limits from Kinesis. + Gauge
    OperatornumberOfAggregatedRecordsPerFetchstream, shardIdThe number of aggregated Kinesis records fetched by the consumer in a single getRecords call to Kinesis. + Gauge
    OperatornumberOfDeggregatedRecordsPerFetchstream, shardIdThe number of deaggregated Kinesis records fetched by the consumer in a single getRecords call to Kinesis. + Gauge
    OperatoraverageRecordSizeBytesstream, shardIdThe average size of a Kinesis record in bytes, fetched by the consumer in a single getRecords call. + Gauge
    OperatorrunLoopTimeNanosstream, shardIdThe actual time taken, in nanoseconds, by the consumer in the run loop. + Gauge
    OperatorloopFrequencyHzstream, shardIdThe number of calls to getRecords in one second. + Gauge
    OperatorbytesRequestedPerFetchstream, shardIdThe bytes requested (2 Mbps / loopFrequencyHz) in a single call to getRecords. + Gauge
    + +### System resources + +System resources reporting is disabled by default. When `metrics.system-resource` +is enabled additional metrics listed below will be available on Job- and TaskManager. +System resources metrics are updated periodically and they present average values for a +configured interval (`metrics.system-resource-probing-interval`). + +System resources reporting requires an optional dependency to be present on the +classpath (for example placed in Flink's `lib` directory): + + - `com.github.oshi:oshi-core:3.4.0` (licensed under EPL 1.0 license) + +Including it's transitive dependencies: + + - `net.java.dev.jna:jna-platform:jar:4.2.2` + - `net.java.dev.jna:jna:jar:4.2.2` + +Failures in this regard will be reported as warning messages like `NoClassDefFoundError` +logged by `SystemResourcesMetricsInitializer` during the startup. + +#### System CPU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ScopeInfixMetricsDescription
    Job-/TaskManagerSystem.CPUUsageOverall % of CPU usage on the machine.
    Idle% of CPU Idle usage on the machine.
    Sys% of System CPU usage on the machine.
    User% of User CPU usage on the machine.
    IOWait% of IOWait CPU usage on the machine.
    Irq% of Irq CPU usage on the machine.
    SoftIrq% of SoftIrq CPU usage on the machine.
    Nice% of Nice Idle usage on the machine.
    Load1minAverage CPU load over 1 minute
    Load5minAverage CPU load over 5 minute
    Load15minAverage CPU load over 15 minute
    UsageCPU*% of CPU usage per each processor
    + +#### System memory + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ScopeInfixMetricsDescription
    Job-/TaskManagerSystem.MemoryAvailableAvailable memory in bytes
    TotalTotal memory in bytes
    System.SwapUsedUsed swap bytes
    TotalTotal swap in bytes
    + +#### System network + + + + + + + + + + + + + + + + + + + + +
    ScopeInfixMetricsDescription
    Job-/TaskManagerSystem.Network.INTERFACE_NAMEReceiveRateAverage receive rate in bytes per second
    SendRateAverage send rate in bytes per second
    ## Latency tracking -Flink allows to track the latency of records traveling through the system. To enable the latency tracking -a `latencyTrackingInterval` (in milliseconds) has to be set to a positive value in the `ExecutionConfig`. +Flink allows to track the latency of records traveling through the system. This feature is disabled by default. +To enable the latency tracking you must set the `latencyTrackingInterval` to a positive number in either the +[Flink configuration]({{ site.baseurl }}/ops/config.html#metrics-latency-interval) or `ExecutionConfig`. At the `latencyTrackingInterval`, the sources will periodically emit a special record, called a `LatencyMarker`. The marker contains a timestamp from the time when the record has been emitted at the sources. @@ -1391,6 +1636,9 @@ latency issues caused by individual machines. Currently, Flink assumes that the clocks of all machines in the cluster are in sync. We recommend setting up an automated clock synchronisation service (like NTP) to avoid false latency results. +Warning Enabling latency metrics can significantly impact the performance +of the cluster. It is highly recommended to only use them for debugging purposes. + ## REST API integration Metrics can be queried through the [Monitoring REST API]({{ site.baseurl }}/monitoring/rest_api.html). @@ -1422,7 +1670,7 @@ Request a list of available metrics: `GET /jobmanager/metrics` -~~~ +{% highlight json %} [ { "id": "metric1" @@ -1431,13 +1679,13 @@ Request a list of available metrics: "id": "metric2" } ] -~~~ +{% endhighlight %} Request the values for specific (unaggregated) metrics: `GET taskmanagers/ABCDE/metrics?get=metric1,metric2` -~~~ +{% highlight json %} [ { "id": "metric1", @@ -1448,13 +1696,13 @@ Request the values for specific (unaggregated) metrics: "value": "2" } ] -~~~ +{% endhighlight %} Request aggregated values for specific metrics: `GET /taskmanagers/metrics?get=metric1,metric2` -~~~ +{% highlight json %} [ { "id": "metric1", @@ -1471,13 +1719,13 @@ Request aggregated values for specific metrics: "sum": 16 } ] -~~~ +{% endhighlight %} Request specific aggregated values for specific metrics: `GET /taskmanagers/metrics?get=metric1,metric2&agg=min,max` -~~~ +{% highlight json %} [ { "id": "metric1", @@ -1490,7 +1738,7 @@ Request specific aggregated values for specific metrics: "max": 14, } ] -~~~ +{% endhighlight %} ## Dashboard integration diff --git a/docs/monitoring/rest_api.md b/docs/monitoring/rest_api.md index 410ba9cacbf4fa..c799e915734f12 100644 --- a/docs/monitoring/rest_api.md +++ b/docs/monitoring/rest_api.md @@ -25,7 +25,7 @@ under the License. Flink has a monitoring API that can be used to query status and statistics of running jobs, as well as recent completed jobs. This monitoring API is used by Flink's own dashboard, but is designed to be used also by custom monitoring tools. -The monitoring API is a REST-ful API that accepts HTTP GET requests and responds with JSON data. +The monitoring API is a REST-ful API that accepts HTTP requests and responds with JSON data. * This will be replaced by the TOC {:toc} @@ -33,653 +33,43 @@ The monitoring API is a REST-ful API that accepts HTTP GET requests and responds ## Overview -The monitoring API is backed by a web server that runs as part of the *JobManager*. By default, this server listens at post `8081`, which can be configured in `flink-conf.yaml` via `jobmanager.web.port`. Note that the monitoring API web server and the web dashboard web server are currently the same and thus run together at the same port. They respond to different HTTP URLs, though. +The monitoring API is backed by a web server that runs as part of the *Dispatcher*. By default, this server listens at post `8081`, which can be configured in `flink-conf.yaml` via `rest.port`. Note that the monitoring API web server and the web dashboard web server are currently the same and thus run together at the same port. They respond to different HTTP URLs, though. -In the case of multiple JobManagers (for high availability), each JobManager will run its own instance of the monitoring API, which offers information about completed and running job while that JobManager was elected the cluster leader. +In the case of multiple Dispatchers (for high availability), each Dispatcher will run its own instance of the monitoring API, which offers information about completed and running job while that Dispatcher was elected the cluster leader. ## Developing -The REST API backend is in the `flink-runtime-web` project. The core class is `org.apache.flink.runtime.webmonitor.WebRuntimeMonitor`, which sets up the server and the request routing. +The REST API backend is in the `flink-runtime` project. The core class is `org.apache.flink.runtime.webmonitor.WebMonitorEndpoint`, which sets up the server and the request routing. We use *Netty* and the *Netty Router* library to handle REST requests and translate URLs. This choice was made because this combination has lightweight dependencies, and the performance of Netty HTTP is very good. -To add new requests, one needs to add a new *request handler* class. A good example to look at is the `org.apache.flink.runtime.webmonitor.handlers.JobExceptionsHandler`. After creating the handler, the handler needs to be registered with the request router in `org.apache.flink.runtime.webmonitor.WebRuntimeMonitor`. +To add new requests, one needs to +* add a new `MessageHeaders` class which serves as an interface for the new request, +* add a new `AbstractRestHandler` class which handles the request according to the added `MessageHeaders` class, +* add the handler to `org.apache.flink.runtime.webmonitor.WebMonitorEndpoint#initializeHandlers()`. +A good example is the `org.apache.flink.runtime.rest.handler.job.JobExceptionsHandler` that uses the `org.apache.flink.runtime.rest.messages.JobExceptionsHeaders`. -## Available Requests -Below is a list of available requests, with a sample JSON response. All requests are of the sample form `http://hostname:8081/jobs`, below we list only the *path* part of the URLs. +## API -Values in angle brackets are variables, for example `http://hostname:8081/jobs//exceptions` will have to requested for example as `http://hostname:8081/jobs/7684be6004e4e955c2a558a9bc463f65/exceptions`. +The REST API is versioned, with specific versions being queryable by prefixing the url with the version prefix. Prefixes are always of the form `v[version_number]`. +For example, to access version 1 of `/foo/bar` one would query `/v1/foo/bar`. - - `/config` - - `/overview` - - `/jobs/overview` - - `/jobs/` - - `/jobs//vertices` - - `/jobs//config` - - `/jobs//exceptions` - - `/jobs//accumulators` - - `/jobs//vertices/` - - `/jobs//vertices//subtasktimes` - - `/jobs//vertices//taskmanagers` - - `/jobs//vertices//accumulators` - - `/jobs//vertices//subtasks/accumulators` - - `/jobs//vertices//subtasks/` - - `/jobs//vertices//subtasks//attempts/` - - `/jobs//vertices//subtasks//attempts//accumulators` - - `/jobs//plan` - - `/jars/upload` - - `/jars` - - `/jars/:jarid` - - `/jars/:jarid/plan` - - `/jars/:jarid/run` +If no version is specified Flink will default to the *oldest* version supporting the request. +Querying unsupported/non-existing versions will return a 404 error. -### General +Attention REST API versioning is *not* active if the cluster runs in [legacy mode](../ops/config.html#mode). For this case please refer to the legacy API below. -**`/config`** +
    -Some information about the monitoring API and the server setup. +
    +#### Dispatcher -Sample Result: +{% include generated/rest_v1_dispatcher.html %} +
    -~~~ -{ - "refresh-interval": 3000, - "timezone-offset": 3600000, - "timezone-name": "Central European Time", - "flink-version": "{{ site.version }}", - "flink-revision": "8124545 @ 16.09.2015 @ 15:38:42 CEST" -} -~~~ +
    -**`/overview`** - -Simple summary of the Flink cluster status. - -Sample Result: - -~~~ -{ - "taskmanagers": 17, - "slots-total": 68, - "slots-available": 68, - "jobs-running": 0, - "jobs-finished": 3, - "jobs-cancelled": 1, - "jobs-failed": 0 -} -~~~ - -### Overview of Jobs - -**`/jobs/overview`** - -Jobs, grouped by status, each with a small summary of its status. - -Sample Result: - -~~~ -{ - "jobs":[ - { - "jid": "7684be6004e4e955c2a558a9bc463f65", - "name": "Flink Java Job at Wed Sep 16 18:08:21 CEST 2015", - "state": "FINISHED", - "start-time": 1442419702857, - "end-time": 1442419975312, - "duration":272455, - "last-modification": 1442419975312, - "tasks": { - "total": 6, - "pending": 0, - "running": 0, - "finished": 6, - "canceling": 0, - "canceled": 0, - "failed": 0 - } - }, - { - "jid": "49306f94d0920216b636e8dd503a6409", - "name": "Flink Java Job at Wed Sep 16 18:16:39 CEST 2015", - ... - }] -} -~~~ - -### Details of a Running or Completed Job - -**`/jobs/`** - -Summary of one job, listing dataflow plan, status, timestamps of state transitions, aggregate information for each vertex (operator). - -Sample Result: - -~~~ -{ - "jid": "ab78dcdbb1db025539e30217ec54ee16", - "name": "WordCount Example", - "state":"FINISHED", - "start-time":1442421277536, - "end-time":1442421299791, - "duration":22255, - "now":1442421991768, - "timestamps": { - "CREATED": 1442421277536, "RUNNING": 1442421277609, "FAILING": 0, "FAILED": 0, "CANCELLING": 0, "CANCELED": 0, "FINISHED": 1442421299791, "RESTARTING": 0 - }, - "vertices": [ { - "id": "19b5b24062c48a06e4eac65422ac3317", - "name": "CHAIN DataSource (at getTextDataSet(WordCount.java:142) ...", - "parallelism": 2, - "status": "FINISHED", - "start-time": 1442421277609, - "end-time": 1442421299469, - "duration": 21860, - "tasks": { - "CREATED": 0, "SCHEDULED": 0, "DEPLOYING": 0, "RUNNING": 0, "FINISHED": 2, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - }, - "metrics": { - "read-bytes": 0, "write-bytes": 37098, "read-records": 0, "write-records": 3312 - } - }, { - "id": "f00c89b349b5c998cfd9fe2a06e50fd0", - "name":"Reduce (SUM(1), at main(WordCount.java:67)", - "parallelism": 2, - .... - }, { - "id": "0a36cbc29102d7bc993d0a9bf23afa12", - "name": "DataSink (CsvOutputFormat (path: /tmp/abzs, delimiter: ))", - ... - } ], - "status-counts": { - "CREATED": 0, "SCHEDULED": 0, "DEPLOYING": 0, "RUNNING": 0, "FINISHED": 3, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - }, - "plan": { - // see plan details below - } -} -~~~ - -**`/jobs//vertices`** - -Currently the same as `/jobs/` - - -**`/jobs//config`** - -The user-defined execution config used by the job. - -Sample Result: - -~~~ -{ - "jid": "ab78dcdbb1db025539e30217ec54ee16", - "name": "WordCount Example", - "execution-config": { - "execution-mode": "PIPELINED", - "restart-strategy": "Restart deactivated", - "job-parallelism": -1, - "object-reuse-mode": false - } -} -~~~ - -**`/jobs//exceptions`** - -The non-recoverable exceptions that have been observed by the job. -The `truncated` flag defines whether more exceptions occurred, but are not listed, because the response would otherwise get too big. - -Sample Result: - -~~~ -{ - "root-exception": "java.io.IOException: File already exists:/tmp/abzs/2\n\tat org.apache.flink.core.fs.local.LocalFileSystem. ...", - "all-exceptions": [ { - "exception": "java.io.IOException: File already exists:/tmp/abzs/1\n\tat org.apache.flink...", - "task": "DataSink (CsvOutputFormat (path: /tmp/abzs, delimiter: )) (1/2)", - "location": "localhost:49220" - }, { - "exception": "java.io.IOException: File already exists:/tmp/abzs/2\n\tat org.apache.flink...", - "task": "DataSink (CsvOutputFormat (path: /tmp/abzs, delimiter: )) (2/2)", - "location": "localhost:49220" - } ], - "truncated":false -} -~~~ - -**`/jobs//accumulators`** - -The aggregated user accumulators plus job accumulators. - -Sample Result: - -~~~ -{ - "job-accumulators":[], - "user-task-accumulators": [ { - "name": "avglen", - "type": "DoubleCounter", - "value": "DoubleCounter 61.5162972" - }, - { - "name": "genwords", - "type": "LongCounter", - "value": "LongCounter 37500000" - } ] -} -~~~ - -**`/jobs//vertices/`** - -Information about one specific vertex, with a summary for each of its subtasks. - -Sample Result: - -~~~ -{ - "id": "dceafe2df1f57a1206fcb907cb38ad97", - "name": "CHAIN DataSource -> Map -> FlatMap -> Combine(SUM(1))", - "parallelism": 2, - "now": 1442424002154, - "subtasks": [ { - "subtask":0, - "status": "FINISHED", - "attempt": 0, - "host": "localhost", - "start-time": 1442421093762, - "end-time": 1442421386680, - "duration": 292918, - "metrics": { - "read-bytes": 0, "write-bytes": 12684375, "read-records": 0, "write-records": 1153125 - } - }, { - "subtask": 1, - "status": "FINISHED", - "attempt": 0, - "host": "localhost", - "start-time": 1442421093774, - "end-time": 1442421386267, - "duration": 292493, - "metrics": { - "read-bytes": 0, "write-bytes": 12684375, "read-records": 0, "write-records": 1153125 - } - } ] -} -~~~ - -**`/jobs//vertices//subtasktimes`** - -This request returns the timestamps for the state transitions of all subtasks of a given vertex. -These can be used, for example, to create time-line comparisons between subtasks. - -Sample Result: - -~~~ -{ - "id": "dceafe2df1f57a1206fcb907cb38ad97", - "name": "CHAIN DataSource -> Map -> Combine(SUM(1))", - "now":1442423745088, - "subtasks": [ { - "subtask": 0, - "host": "localhost", - "duration": 292924, - "timestamps": { - "CREATED": 1442421093741, "SCHEDULED": 1442421093756, "DEPLOYING": 1442421093762, "RUNNING": 1442421094026, "FINISHED": 1442421386680, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - } - }, { - "subtask": 1, - "host": "localhost", - "duration": 292494, - "timestamps": { - "CREATED": 1442421093741, "SCHEDULED": 1442421093773, "DEPLOYING": 1442421093774, "RUNNING": 1442421094013, "FINISHED": 1442421386267, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - } - } ] -} -~~~ - -**`/jobs//vertices//taskmanagers`** - -TaskManager statistics for one specific vertex. This is an aggregation of subtask statistics returned by `/jobs//vertices/`. - -Sample Result: - -~~~ -{ - "id": "fe20bcc29b87cdc76589ca42114c2499", - "name": "Reduce (SUM(1), at main(WordCount.java:72)", - "now": 1454348282653, - "taskmanagers": [ { - "host": "ip-10-0-43-227:35413", - "status": "FINISHED", - "start-time": 1454347870991, - "end-time": 1454347872111, - "duration": 1120, - "metrics": { - "read-bytes": 32503056, "write-bytes": 9637041, "read-records": 2906087, "write-records": 849467 - }, - "status-counts": { - "CREATED": 0, "SCHEDULED": 0, "DEPLOYING": 0, "RUNNING": 0, "FINISHED": 18, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - } - },{ - "host": "ip-10-0-43-227:41486", - "status": "FINISHED", - "start-time": 1454347871001, - "end-time": 1454347872395, - "duration": 1394, - "metrics": { - "read-bytes": 32389499, "write-bytes": 9608829, "read-records": 2895999, "write-records": 846948 - }, - "status-counts": { - "CREATED": 0, "SCHEDULED": 0, "DEPLOYING": 0, "RUNNING": 0, "FINISHED": 18, "CANCELING": 0, "CANCELED": 0, "FAILED": 0 - } - } ] -} -~~~ - -**`/jobs//vertices//accumulators`** - -The aggregated user-defined accumulators, for a specific vertex. - -Sample Result: - -~~~ -{ - "id": "dceafe2df1f57a1206fcb907cb38ad97", - "user-accumulators": [ { - "name": "avglen", "type": "DoubleCounter", "value": "DoubleCounter 123.03259440000001" - }, { - "name": "genwords", "type": "LongCounter", "value": "LongCounter 75000000" - } ] -} -~~~ - -**`/jobs//vertices//subtasks/accumulators`** - -Gets all user-defined accumulators for all subtasks of a given vertex. These are the individual accumulators that are returned in aggregated form by the -request `/jobs//vertices//accumulators`. - -Sample Result: - -~~~ -{ - "id": "dceafe2df1f57a1206fcb907cb38ad97", - "parallelism": 2, - "subtasks": [ { - "subtask": 0, - "attempt": 0, - "host": "localhost", - "user-accumulators": [ { - "name": "genwords", "type": "LongCounter", "value": "LongCounter 62500000" - }, { - "name": "genletters", "type": "LongCounter", "value": "LongCounter 1281589525" - } ] - }, { - "subtask": 1, - "attempt": 0, - "host": "localhost", - "user-accumulators": [ { - "name": "genwords", "type": "LongCounter", "value": "LongCounter 12500000" - }, { - "name": "genletters", "type": "LongCounter", "value": "LongCounter 256317905" - } ] - } ] -} -~~~ - -**`/jobs//vertices//subtasks/`** - -Summary of the current or latest execution attempt of a specific subtask. See below for a sample. - - -**`/jobs//vertices//subtasks//attempts/`** - -Summary of a specific execution attempt of a specific subtask. Multiple execution attempts happen in case of failure/recovery. - -Sample Result: - -~~~ -{ - "subtask": 0, - "status": "FINISHED", - "attempt": 0, - "host": "localhost", - "start-time": 1442421093762, - "end-time": 1442421386680, - "duration": 292918, - "metrics": { - "read-bytes": 0, "write-bytes": 12684375, "read-records": 0, "write-records": 1153125 - } -} -~~~ - -**`/jobs//vertices//subtasks//attempts//accumulators`** - -The accumulators collected for one specific subtask during one specific execution attempt (multiple attempts happen in case of failure/recovery). - -Sample Result: - -~~~ -{ - "subtask": 0, - "attempt": 0, - "id": "b22f94d91bf41ddb", - "user-accumulators": [ { - "name": "genwords", "type":"LongCounter", "value":"LongCounter 62500000" - }, { - "name": "genletters", "type": "LongCounter", "value": "LongCounter 1281589525" - }, { - "name": "avglen", "type": "DoubleCounter", "value": "DoubleCounter 102.527162" - } ] -} -~~~ - -**`/jobs//plan`** - -The dataflow plan of a job. The plan is also included in the job summary (`/jobs/`). - -Sample Result: - -~~~ -{ - "jid":"ab78dcdbb1db025539e30217ec54ee16", - "name":"WordCount Example", - "nodes": [ { - "id": "f00c89b349b5c998cfd9fe2a06e50fd0", - "parallelism": 2, - "operator": "GroupReduce", - "operator_strategy": "Sorted Group Reduce", - "description": "Reduce (SUM(1), at main(WordCount.java:67)", - "inputs": [ { - "num": 0, - "id":"19b5b24062c48a06e4eac65422ac3317", - "ship_strategy": "Hash Partition on [0]", - "local_strategy":"Sort (combining) on [0:ASC]", - "exchange":"pipelined" - } ], - "optimizer_properties": { - "global_properties": [ - { "name":"Partitioning", "value":"HASH_PARTITIONED" }, - { "name":"Partitioned on", "value":"[0]" }, - { "name":"Partitioning Order", "value":"(none)" }, - { "name":"Uniqueness", "value":"not unique" } - ], - "local_properties": [ - { "name":"Order", "value":"[0:ASC]" }, - { "name":"Grouped on", "value":"[0]" }, - { "name":"Uniqueness", "value":"not unique" } - ], - "estimates": [ - { "name":"Est. Output Size", "value":"(unknown)" }, - { "name":"Est. Cardinality", "value":"(unknown)" } - ], - "costs": [ - { "name":"Network", "value":"(unknown)" }, - { "name":"Disk I/O", "value":"(unknown)" }, - { "name":"CPU", "value":"(unknown)" }, - { "name":"Cumulative Network", "value":"(unknown)" }, - { "name":"Cumulative Disk I/O", "value":"(unknown)" }, - { "name":"Cumulative CPU","value":"(unknown)" } - ], - "compiler_hints": [ - { "name":"Output Size (bytes)", "value":"(none)" }, - { "name":"Output Cardinality", "value":"(none)" }, - { "name":"Avg. Output Record Size (bytes)", "value":"(none)" }, - { "name":"Filter Factor", "value":"(none)" } - ] - } - }, - { - "id": "19b5b24062c48a06e4eac65422ac3317", - "parallelism": 2, - "operator": "Data Source -> FlatMap -> GroupCombine", - "operator_strategy":" (none) -> FlatMap -> Sorted Combine", - "description":"DataSource (at getTextDataSet(WordCount.java:142) (org.apache.flink.api.java.io.TextInputFormat)) -> FlatMap (FlatMap at main(WordCount.java:67)) -> Combine(SUM(1), at main(WordCount.java:67)", - "optimizer_properties": { - ... - } - }, - { - "id": "0a36cbc29102d7bc993d0a9bf23afa12", - "parallelism": 2, - "operator": "Data Sink", - "operator_strategy": "(none)", - "description": "DataSink (CsvOutputFormat (path: /tmp/abzs, delimiter: ))", - "inputs":[ { - "num": 0, - "id": "f00c89b349b5c998cfd9fe2a06e50fd0", - "ship_strategy": "Forward", - "exchange": "pipelined" - } ], - "optimizer_properties": { - ... - } - } ] -} -~~~ - -### Job Cancellation - -#### Cancel Job - -`DELETE` request to **`/jobs/:jobid/cancel`**. - -Triggers job cancellation, result on success is `{}`. - -#### Cancel Job with Savepoint - -Triggers a savepoint and cancels the job after the savepoint succeeds. - -`GET` request to **`/jobs/:jobid/cancel-with-savepoint/`** triggers a savepoint to the default savepoint directory and cancels the job. - -`GET` request to **`/jobs/:jobid/cancel-with-savepoint/target-directory/:targetDirectory`** triggers a savepoint to the given target directory and cancels the job. - -Since savepoints can take some time to complete this operation happens asynchronously. The result to this request is the location of the in-progress cancellation. - -Sample Trigger Result: - -~~~ -{ - "status": "accepted", - "request-id": 1, - "location": "/jobs/:jobid/cancel-with-savepoint/in-progress/1" -} -~~~ - -##### Monitoring Progress - -The progress of the cancellation has to be monitored by the user at - -~~~ -/jobs/:jobid/cancel-with-savepoint/in-progress/:requestId -~~~ - -The request ID is returned by the trigger result. - -###### In-Progress - -~~~ -{ - "status": "in-progress", - "request-id": 1 -} -~~~ - -###### Success - -~~~ -{ - "status": "success", - "request-id": 1, - "savepoint-path": "" -} -~~~ - -The `savepointPath` points to the external path of the savepoint, which can be used to resume the savepoint. - -###### Failed - -~~~ -{ - "status": "failed", - "request-id": 1, - "cause": "" -} -~~~ - -### Submitting Programs - -It is possible to upload, run, and list Flink programs via the REST APIs and web frontend. - -#### Upload a new JAR file - -Send a `POST` request to `/jars/upload` with your jar file sent as multi-part data under the `jarfile` file. -Also make sure that the multi-part data includes the `Content-Type` of the file itself, some http libraries do not add the header by default. - -The multi-part payload should start like - -``` -------BoundaryXXXX -Content-Disposition: form-data; name="jarfile"; filename="YourFileName.jar" -Content-Type: application/x-java-archive -``` - -#### Run a Program (POST) - -Send a `POST` request to `/jars/:jarid/run`. The `jarid` parameter is the file name of the program JAR in the configured web frontend upload directory (configuration key `jobmanager.web.upload.dir`). - -You can specify the following query parameters (all optional): - -- **Program arguments**: `program-args=arg1 arg2 arg3` -- **Main class to execute**: `entry-class=EntryClassName.class` -- **Default parallelism**: `parallelism=4` -- **Savepoint path to restore from**: `savepointPath=hdfs://path/to/savepoint` -- **Allow non restored state**: `allowNonRestoredState=true` - -If the call succeeds, you will get a response with the ID of the submitted job. - -**Example:** Run program with a savepoint - -Request: - -~~~ -POST: /jars/MyProgram.jar/run?savepointPath=/my-savepoints/savepoint-1bae02a80464&allowNonRestoredState=true -~~~ - -Response: - -~~~ -{"jobid": "869a9868d49c679e7355700e0857af85"} -~~~ - -## FLIP-6 - -The following is the REST API documentation for FLIP-6. - -### Dispatcher - -{% include generated/rest_dispatcher.html %} - -{% top %} diff --git a/docs/ops/cli.md b/docs/ops/cli.md index 45fe799ccbbc07..ec3596fa55d2b5 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -23,55 +23,57 @@ specific language governing permissions and limitations under the License. --> -Flink provides a command-line interface to run programs that are packaged -as JAR files, and control their execution. The command line interface is part +Flink provides a Command-Line Interface (CLI) to run programs that are packaged +as JAR files, and control their execution. The CLI is part of any Flink setup, available in local single node setups and in distributed setups. It is located under `/bin/flink` and connects by default to the running Flink master (JobManager) that was started from the same installation directory. -A prerequisite to using the command line interface is that the Flink -master (JobManager) has been started (via -`/bin/start-cluster.sh`) or that a YARN environment is -available. - The command line can be used to - submit jobs for execution, - cancel a running job, -- provide information about a job, and -- list running and waiting jobs. +- provide information about a job, +- list running and waiting jobs, +- trigger and dispose savepoints, and +- modify a running job + +A prerequisite to using the command line interface is that the Flink +master (JobManager) has been started (via +`/bin/start-cluster.sh`) or that a YARN environment is +available. * This will be replaced by the TOC {:toc} ## Examples -- Run example program with no arguments. +- Run example program with no arguments: ./bin/flink run ./examples/batch/WordCount.jar -- Run example program with arguments for input and result files +- Run example program with arguments for input and result files: ./bin/flink run ./examples/batch/WordCount.jar \ --input file:///home/user/hamlet.txt --output file:///home/user/wordcount_out -- Run example program with parallelism 16 and arguments for input and result files +- Run example program with parallelism 16 and arguments for input and result files: ./bin/flink run -p 16 ./examples/batch/WordCount.jar \ --input file:///home/user/hamlet.txt --output file:///home/user/wordcount_out -- Run example program with flink log output disabled +- Run example program with flink log output disabled: ./bin/flink run -q ./examples/batch/WordCount.jar -- Run example program in detached mode +- Run example program in detached mode: ./bin/flink run -d ./examples/batch/WordCount.jar - Run example program on a specific JobManager: - ./bin/flink run -m myJMHost:6123 \ + ./bin/flink run -m myJMHost:8081 \ ./examples/batch/WordCount.jar \ --input file:///home/user/hamlet.txt --output file:///home/user/wordcount_out @@ -104,6 +106,10 @@ The command line can be used to ./bin/flink list -r +- List all existing jobs (including their JobIDs): + + ./bin/flink list -a + - List running Flink jobs inside Flink YARN session: ./bin/flink list -m yarn-cluster -yid -r @@ -119,9 +125,13 @@ The command line can be used to - Stop a job (streaming jobs only): ./bin/flink stop + +- Modify a running job (streaming jobs only): + ./bin/flink modify -p -The difference between cancelling and stopping a (streaming) job is the following: + +**NOTE**: The difference between cancelling and stopping a (streaming) job is the following: On a cancel call, the operators in a job immediately receive a `cancel()` method call to cancel them as soon as possible. @@ -148,7 +158,7 @@ This will trigger a savepoint for the job with ID `jobId`, and returns the path Furthermore, you can optionally specify a target file system directory to store the savepoint in. The directory needs to be accessible by the JobManager. -If you don't specify a target directory, you need to have [configured a default directory](#configuration) (see [Savepoints]({{site.baseurl}}/ops/state/savepoints.html#configuration)). Otherwise, triggering the savepoint will fail. +If you don't specify a target directory, you need to have [configured a default directory]({{site.baseurl}}/ops/state/savepoints.html#configuration). Otherwise, triggering the savepoint will fail. #### Trigger a Savepoint with YARN @@ -165,7 +175,7 @@ Everything else is the same as described in the above **Trigger a Savepoint** se You can atomically trigger a savepoint and cancel a job. {% highlight bash %} -./bin/flink cancel -s [savepointDirectory] +./bin/flink cancel -s [savepointDirectory] {% endhighlight %} If no savepoint directory is configured, you need to configure a default savepoint directory for the Flink installation (see [Savepoints]({{site.baseurl}}/ops/state/savepoints.html#configuration)). @@ -208,7 +218,7 @@ Otherwise, you will run into a `ClassNotFoundException`. The command line syntax is as follows: -~~~ +{% highlight bash %} ./flink [OPTIONS] [ARGUMENTS] The following actions are available: @@ -217,59 +227,64 @@ Action "run" compiles and runs a program. Syntax: run [OPTIONS] "run" action options: - -c,--class Class with the program entry - point ("main" method or - "getPlan()" method. Only - needed if the JAR file does - not specify the class in its - manifest. - -C,--classpath Adds a URL to each user code - classloader on all nodes in - the cluster. The paths must - specify a protocol (e.g. - file://) and be accessible - on all nodes (e.g. by means - of a NFS share). You can use - this option multiple times - for specifying more than one - URL. The protocol must be - supported by the {@link - java.net.URLClassLoader}. - -d,--detached If present, runs the job in - detached mode - -m,--jobmanager Address of the JobManager - (master) to which to - connect. Use this flag to - connect to a different - JobManager than the one - specified in the - configuration. - -n,--allowNonRestoredState Allow non restored savepoint - state in case an operator has - been removed from the job. - -p,--parallelism The parallelism with which - to run the program. Optional - flag to override the default - value specified in the - configuration. - -q,--sysoutLogging If present, suppress logging - output to standard out. - -s,--fromSavepoint Path to a savepoint to - restore the job from (for - example - hdfs:///flink/savepoint-1537 - ). - -z,--zookeeperNamespace Namespace to create the - Zookeeper sub-paths for high - availability mode - + -c,--class Class with the program entry point + ("main" method or "getPlan()" method. + Only needed if the JAR file does not + specify the class in its manifest. + -C,--classpath Adds a URL to each user code + classloader on all nodes in the + cluster. The paths must specify a + protocol (e.g. file://) and be + accessible on all nodes (e.g. by means + of a NFS share). You can use this + option multiple times for specifying + more than one URL. The protocol must + be supported by the {@link + java.net.URLClassLoader}. + -d,--detached If present, runs the job in detached + mode + -n,--allowNonRestoredState Allow to skip savepoint state that + cannot be restored. You need to allow + this if you removed an operator from + your program that was part of the + program when the savepoint was + triggered. + -p,--parallelism The parallelism with which to run the + program. Optional flag to override the + default value specified in the + configuration. + -q,--sysoutLogging If present, suppress logging output to + standard out. + -s,--fromSavepoint Path to a savepoint to restore the job + from (for example + hdfs:///flink/savepoint-1537). + -sae,--shutdownOnAttachedExit If the job is submitted in attached + mode, perform a best-effort cluster + shutdown when the CLI is terminated + abruptly, e.g., in response to a user + interrupt, such as typing Ctrl + C. Options for yarn-cluster mode: - -yD Dynamic properties - -yd,--yarndetached Start detached + -d,--detached If present, runs the job in detached + mode + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to + connect to a different JobManager than + the one specified in the + configuration. + -sae,--shutdownOnAttachedExit If the job is submitted in attached + mode, perform a best-effort cluster + shutdown when the CLI is terminated + abruptly, e.g., in response to a user + interrupt, such as typing Ctrl + C. + -yD use value for given property + -yd,--yarndetached If present, runs the job in detached + mode (deprecated; use non-YARN + specific option instead) + -yh,--yarnhelp Help for the Yarn session CLI. -yid,--yarnapplicationId Attach to running YARN session -yj,--yarnjar Path to Flink jar file - -yjm,--yarnjobManagerMemory Memory for JobManager Container [in - MB] + -yjm,--yarnjobManagerMemory Memory for JobManager Container + with optional unit (default: MB) -yn,--yarncontainer Number of YARN container to allocate (=Number of Task Managers) -ynm,--yarnname Set a custom name for the application @@ -280,11 +295,24 @@ Action "run" compiles and runs a program. -ys,--yarnslots Number of slots per TaskManager -yst,--yarnstreaming Start Flink in streaming mode -yt,--yarnship Ship files in the specified directory - (t for transfer) - -ytm,--yarntaskManagerMemory Memory per TaskManager Container [in - MB] + (t for transfer), multiple options are + supported. + -ytm,--yarntaskManagerMemory Memory per TaskManager Container + with optional unit (default: MB) -yz,--yarnzookeeperNamespace Namespace to create the Zookeeper sub-paths for high availability mode + -ynl,--yarnnodeLabel Specify YARN node label for + the YARN application + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode @@ -300,23 +328,30 @@ Action "info" shows the optimized execution plan of the program (JSON). program. Optional flag to override the default value specified in the configuration. - Options for yarn-cluster mode: - -yid,--yarnapplicationId Attach to running YARN session - Action "list" lists running and scheduled programs. Syntax: list [OPTIONS] "list" action options: - -m,--jobmanager Address of the JobManager (master) to which - to connect. Use this flag to connect to a - different JobManager than the one specified - in the configuration. - -r,--running Show only running programs and their JobIDs - -s,--scheduled Show only scheduled programs and their JobIDs + -r,--running Show only running programs and their JobIDs + -s,--scheduled Show only scheduled programs and their JobIDs Options for yarn-cluster mode: + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to connect + to a different JobManager than the one + specified in the configuration. -yid,--yarnapplicationId Attach to running YARN session + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode @@ -324,12 +359,23 @@ Action "stop" stops a running program (streaming jobs only). Syntax: stop [OPTIONS] "stop" action options: - -m,--jobmanager Address of the JobManager (master) to which - to connect. Use this flag to connect to a - different JobManager than the one specified - in the configuration. + Options for yarn-cluster mode: + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to connect + to a different JobManager than the one + specified in the configuration. -yid,--yarnapplicationId Attach to running YARN session + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode @@ -337,18 +383,27 @@ Action "cancel" cancels a running program. Syntax: cancel [OPTIONS] "cancel" action options: - -m,--jobmanager Address of the JobManager (master) - to which to connect. Use this flag - to connect to a different JobManager - than the one specified in the - configuration. -s,--withSavepoint Trigger savepoint and cancel job. The target directory is optional. If no directory is specified, the configured default directory (state.savepoints.dir) is used. Options for yarn-cluster mode: + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to connect + to a different JobManager than the one + specified in the configuration. -yid,--yarnapplicationId Attach to running YARN session + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode @@ -356,14 +411,51 @@ Action "savepoint" triggers savepoints for a running job or disposes existing on Syntax: savepoint [OPTIONS] [] "savepoint" action options: - -d,--dispose Path of savepoint to dispose. - -j,--jarfile Flink program JAR file. - -m,--jobmanager Address of the JobManager (master) to which - to connect. Use this flag to connect to a - different JobManager than the one specified - in the configuration. + -d,--dispose Path of savepoint to dispose. + -j,--jarfile Flink program JAR file. Options for yarn-cluster mode: + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to connect + to a different JobManager than the one + specified in the configuration. -yid,--yarnapplicationId Attach to running YARN session -~~~ + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode + + + +Action "modify" modifies a running job (e.g. change of parallelism). + + Syntax: modify [OPTIONS] + "modify" action options: + -h,--help Show the help message for the CLI + Frontend or the action. + -p,--parallelism New parallelism for the specified job. + -v,--verbose This option is deprecated. + Options for yarn-cluster mode: + -m,--jobmanager Address of the JobManager (master) to + which to connect. Use this flag to connect + to a different JobManager than the one + specified in the configuration. + -yid,--yarnapplicationId Attach to running YARN session + -z,--zookeeperNamespace Namespace to create the Zookeeper + sub-paths for high availability mode + + Options for default mode: + -m,--jobmanager Address of the JobManager (master) to which + to connect. Use this flag to connect to a + different JobManager than the one specified + in the configuration. + -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths + for high availability mode +{% endhighlight %} {% top %} diff --git a/docs/ops/config.md b/docs/ops/config.md index efd3ee3af90370..79e768c3946031 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -40,211 +40,7 @@ The configuration files for the TaskManagers can be different, Flink does not as ## Common Options -- `env.java.home`: The path to the Java installation to use (DEFAULT: system's default Java installation, if found). Needs to be specified if the startup scripts fail to automatically resolve the java home directory. Can be specified to point to a specific java installation or version. If this option is not specified, the startup scripts also evaluate the `$JAVA_HOME` environment variable. - -- `env.java.opts`: Set custom JVM options. This value is respected by Flink's start scripts, both JobManager and -TaskManager, and Flink's YARN client. This can be used to set different garbage collectors or to include remote -debuggers into the JVMs running Flink's services. Enclosing options in double quotes delays parameter substitution -allowing access to variables from Flink's startup scripts. Use `env.java.opts.jobmanager` and `env.java.opts.taskmanager` -for JobManager or TaskManager-specific options, respectively. - -- `env.java.opts.jobmanager`: JobManager-specific JVM options. These are used in addition to the regular `env.java.opts`. - -- `env.java.opts.taskmanager`: TaskManager-specific JVM options. These are used in addition to the regular `env.java.opts`. - -- `jobmanager.rpc.address`: The external address of the JobManager, which is the master/coordinator of the distributed system (DEFAULT: localhost). **Note:** The address (host name or IP) should be accessible by all nodes including the client. - -- `jobmanager.rpc.port`: The port number of the JobManager (DEFAULT: 6123). - -- `jobmanager.heap.mb`: JVM heap size (in megabytes) for the JobManager. You may have to increase the heap size for the JobManager if you are running very large applications (with many operators), or if you are keeping a long history of them. - -- `taskmanager.heap.mb`: JVM heap size (in megabytes) for the TaskManagers, which are the parallel workers of the system. In contrast to Hadoop, Flink runs operators (e.g., join, aggregate) and user-defined functions (e.g., Map, Reduce, CoGroup) inside the TaskManager (including sorting/hashing/caching), so this value should be as large as possible. If the cluster is exclusively running Flink, the total amount of available memory per machine minus some memory for the operating system (maybe 1-2 GB) is a good value. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value. - -- `taskmanager.numberOfTaskSlots`: The number of parallel operator or user function instances that a single TaskManager can run (DEFAULT: 1). If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores). [More about task slots](config.html#configuring-taskmanager-processing-slots). - -- `parallelism.default`: The default parallelism to use for programs that have no parallelism specified. (DEFAULT: 1). For setups that have no concurrent jobs running, setting this value to NumTaskManagers * NumSlotsPerTaskManager will cause the system to use all available execution resources for the program's execution. **Note**: The default parallelism can be overwritten for an entire job by calling `setParallelism(int parallelism)` on the `ExecutionEnvironment` or by passing `-p ` to the Flink Command-line frontend. It can be overwritten for single transformations by calling `setParallelism(int -parallelism)` on an operator. See [Parallel Execution]({{site.baseurl}}/dev/parallel.html) for more information about parallelism. - -- `fs.default-scheme`: The default filesystem scheme to be used, with the necessary authority to contact, e.g. the host:port of the NameNode in the case of HDFS (if needed). -By default, this is set to `file:///` which points to the local filesystem. This means that the local -filesystem is going to be used to search for user-specified files **without** an explicit scheme -definition. As another example, if this is set to `hdfs://localhost:9000/`, then a user-specified file path -without explicit scheme definition, such as `/user/USERNAME/in.txt`, is going to be transformed into -`hdfs://localhost:9000/user/USERNAME/in.txt`. This scheme is used **ONLY** if no other scheme is specified (explicitly) in the user-provided `URI`. - -- `classloader.resolve-order`: Whether Flink should use a child-first `ClassLoader` when loading -user-code classes or a parent-first `ClassLoader`. Can be one of `parent-first` or `child-first`. (default: `child-first`) - -- `classloader.parent-first-patterns`: A (semicolon-separated) list of patterns that specifies which -classes should always be resolved through the parent `ClassLoader` first. A pattern is a simple -prefix that is checked against the fully qualified class name. By default, this is set to -`java.;org.apache.flink.;javax.annotation;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback`. -If you want to change this setting you have to make sure to also include the default patterns in -your list of patterns if you want to keep that default behaviour. - -## Advanced Options - -### Compute - -- `taskmanager.compute.numa`: When enabled a TaskManager is started on each NUMA node for each worker listed in *conf/slaves* (DEFAULT: false). Note: only supported when deploying Flink as a standalone cluster. - -### Managed Memory - -By default, Flink allocates a fraction of `0.7` of the free memory (total memory configured via `taskmanager.heap.mb` minus memory used for network buffers) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents `OutOfMemoryException`s because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize the data to convert it into Java objects. All in all, managed memory improves the robustness and speed of the system. - -The default fraction for managed memory can be adjusted using the `taskmanager.memory.fraction` parameter. An absolute value may be set using `taskmanager.memory.size` (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes. - -- `taskmanager.memory.size`: The amount of memory (in megabytes) that the task manager reserves on-heap or off-heap (depending on `taskmanager.memory.off-heap`) for sorting, hash tables, and caching of intermediate results. If unspecified (-1), the memory manager will take a fixed ratio with respect to the size of the task manager JVM as specified by `taskmanager.memory.fraction`. (DEFAULT: -1) - -- `taskmanager.memory.fraction`: The relative amount of memory (with respect to `taskmanager.heap.mb`, after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on `taskmanager.memory.off-heap`) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. (DEFAULT: 0.7) This parameter is only evaluated, if `taskmanager.memory.size` is not set. - -- `taskmanager.memory.off-heap`: If set to `true`, the task manager allocates memory which is used for sorting, hash tables, and caching of intermediate results outside of the JVM heap. For setups with larger quantities of memory, this can improve the efficiency of the operations performed on the memory (DEFAULT: false). - -- `taskmanager.memory.segment-size`: The size of memory buffers used by the memory manager and the network stack in bytes (DEFAULT: 32768 (= 32 KiBytes)). - -- `taskmanager.memory.preallocate`: Can be either of `true` or `false`. Specifies whether task managers should allocate all managed memory when starting up. (DEFAULT: false). When `taskmanager.memory.off-heap` is set to `true`, then it is advised that this configuration is also set to `true`. If this configuration is set to `false` cleaning up of the allocated offheap memory happens only when the configured JVM parameter MaxDirectMemorySize is reached by triggering a full GC. **Note:** For streaming setups, we highly recommend to set this value to `false` as the core state backends currently do not use the managed memory. - -### Memory and Performance Debugging - -These options are useful for debugging a Flink application for memory and garbage collection related issues, such as performance and out-of-memory process kills or exceptions. - -- `taskmanager.debug.memory.startLogThread`: Causes the TaskManagers to periodically log memory and Garbage collection statistics. The statistics include current heap-, off-heap, and other memory pool utilization, as well as the time spent on garbage collection, by heap memory pool. - -- `taskmanager.debug.memory.logIntervalMs`: The interval (in milliseconds) in which the TaskManagers log the memory and garbage collection statistics. Only has an effect, if `taskmanager.debug.memory.startLogThread` is set to true. - -### Kerberos-based Security - -Flink supports Kerberos authentication for the following services: - -+ Hadoop Components, such as HDFS, YARN, or HBase *(version 2.6.1 and above; all other versions have critical bugs which might fail the Flink job unexpectedly)*. -+ Kafka Connectors *(version 0.9+ and above)*. -+ Zookeeper - -Configuring Flink for Kerberos security involves three aspects, explained separately in the following sub-sections. - -##### 1. Providing the cluster with a Kerberos credential (i.e. a keytab or a ticket via `kinit`) - -To provide the cluster with a Kerberos credential, Flink supports using a Kerberos keytab file or ticket caches managed by `kinit`. - -- `security.kerberos.login.use-ticket-cache`: Indicates whether to read from your Kerberos ticket cache (default: `true`). - -- `security.kerberos.login.keytab`: Absolute path to a Kerberos keytab file that contains the user credentials. - -- `security.kerberos.login.principal`: Kerberos principal name associated with the keytab. - -If both `security.kerberos.login.keytab` and `security.kerberos.login.principal` have values provided, keytabs will be used for authentication. -It is preferable to use keytabs for long-running jobs, to avoid ticket expiration issues. If you prefer to use the ticket cache, -talk to your administrator about increasing the Hadoop delegation token lifetime. - -Note that authentication using ticket caches is only supported when deploying Flink as a standalone cluster or on YARN. - -##### 2. Making the Kerberos credential available to components and connectors as needed - -For Hadoop components, Flink will automatically detect if the configured Kerberos credentials should be used when connecting to HDFS, HBase, and other Hadoop components depending on whether Hadoop security is enabled (in `core-site.xml`). - -For any connector or component that uses a JAAS configuration file, make the Kerberos credentials available to them by configuring JAAS login contexts for each one respectively, using the following configuration: - -- `security.kerberos.login.contexts`: A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication). - -This allows enabling Kerberos authentication for different connectors or components independently. For example, you can enable Hadoop security without necessitating the use of Kerberos for ZooKeeper, or vice versa. - -You may also provide a static JAAS configuration file using the mechanisms described in the [Java SE Documentation](http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html), whose entries will override those produced by the above configuration option. - -##### 3. Configuring the component and/or connector to use Kerberos authentication - -Finally, be sure to configure the connector within your Flink program or component as necessary to use Kerberos authentication. - -Below is a list of currently first-class supported connectors or components by Flink for Kerberos authentication: - -- Kafka: see [here]({{site.baseurl}}/dev/connectors/kafka.html#enabling-kerberos-authentication-for-versions-above-09-only) for details on configuring the Kafka connector to use Kerberos authentication. - -- Zookeeper (for HA): see [here]({{site.baseurl}}/ops/jobmanager_high_availability.html#configuring-for-zookeeper-security) for details on Zookeeper security configuration to work with the Kerberos-based security configurations mentioned here. - -For more information on how Flink security internally setups Kerberos authentication, please see [here]({{site.baseurl}}/ops/security-kerberos.html). - -### Other - -- `taskmanager.tmp.dirs`: The directory for temporary files, or a list of directories separated by the system's directory delimiter (for example ':' (colon) on Linux/Unix). If multiple directories are specified, then the temporary files will be distributed across the directories in a round-robin fashion. The I/O manager component will spawn one reading and one writing thread per directory. A directory may be listed multiple times to have the I/O manager use multiple threads for it (for example if it is physically stored on a very fast disc or RAID) (DEFAULT: The system's tmp dir). - -- `taskmanager.log.path`: The config parameter defining the taskmanager log file location - -- `jobmanager.web.address`: Address of the JobManager's web interface (DEFAULT: anyLocalAddress()). - -- `jobmanager.web.port`: Port of the JobManager's web interface (DEFAULT: 8081). - -- `jobmanager.web.tmpdir`: This configuration parameter allows defining the Flink web directory to be used by the web interface. The web interface -will copy its static files into the directory. Also uploaded job jars are stored in the directory if not overridden. By default, the temporary directory is used. - -- `jobmanager.web.upload.dir`: The config parameter defining the directory for uploading the job jars. If not specified a dynamic directory -will be used under the directory specified by jobmanager.web.tmpdir. - -- `fs.overwrite-files`: Specifies whether file output writers should overwrite existing files by default. Set to *true* to overwrite by default, *false* otherwise. (DEFAULT: false) - -- `fs.output.always-create-directory`: File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to *true*, writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to *false*, the writer will directly create the file directly at the output path, without creating a containing directory. (DEFAULT: false) - -- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. (DEFAULT: 0.1) - -- `taskmanager.network.memory.min`: Minimum memory size for network buffers in bytes (DEFAULT: 64 MB) - -- `taskmanager.network.memory.max`: Maximum memory size for network buffers in bytes (DEFAULT: 1 GB) - -- `state.backend`: The backend that will be used to store operator state checkpoints if checkpointing is enabled. Supported backends: - - `jobmanager`: In-memory state, backup to JobManager's/ZooKeeper's memory. Should be used only for minimal state (Kafka offsets) or testing and local debugging. - - `filesystem`: State is in-memory on the TaskManagers, and state snapshots are stored in a file system. Supported are all filesystems supported by Flink, for example HDFS, S3, ... - -- `state.backend.fs.checkpointdir`: Directory for storing checkpoints in a Flink supported filesystem. Note: State backend must be accessible from the JobManager, use `file://` only for local setups. - -- `state.backend.rocksdb.checkpointdir`: The local directory for storing RocksDB files, or a list of directories separated by the systems directory delimiter (for example ':' (colon) on Linux/Unix). (DEFAULT value is `taskmanager.tmp.dirs`) - -- `state.checkpoints.dir`: The target directory for meta data of [externalized checkpoints]({{ site.baseurl }}/ops/state/checkpoints.html#externalized-checkpoints). - -- `state.checkpoints.num-retained`: The number of completed checkpoint instances to retain. Having more than one allows recovery fallback to an earlier checkpoints if the latest checkpoint is corrupt. (Default: 1) - -- `high-availability.zookeeper.storageDir`: Required for HA. Directory for storing JobManager metadata; this is persisted in the state backend and only a pointer to this state is stored in ZooKeeper. Exactly like the checkpoint directory it must be accessible from the JobManager and a local filesystem should only be used for local deployments. Previously this key was named `recovery.zookeeper.storageDir`. - -- `blob.storage.directory`: Directory for storing blobs (such as user JARs) on the TaskManagers. -If not set or empty, Flink will fall back to `taskmanager.tmp.dirs` and select one temp directory -at random. - -- `blob.service.cleanup.interval`: Cleanup interval (in seconds) of transient blobs at server and caches as well as permanent blobs at the caches (DEFAULT: 1 hour). -Whenever a job is not referenced at the cache anymore, we set a TTL for its permanent blob files and -let the periodic cleanup task (executed every `blob.service.cleanup.interval` seconds) remove them -after this TTL has passed. We do the same for transient blob files at both server and caches but -immediately after accessing them, i.e. an put or get operation. -This means that a blob will be retained at most 2 * `blob.service.cleanup.interval` seconds after -not being referenced anymore (permanent blobs) or their last access (transient blobs). For permanent blobs, -this means that a recovery still has the chance to use existing files rather downloading them again. - -- `blob.server.port`: Port definition for the blob server (serving user JARs) on the TaskManagers. By default the port is set to 0, which means that the operating system is picking an ephemeral port. Flink also accepts a list of ports ("50100,50101"), ranges ("50100-50200") or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple JobManagers are running on the same machine. - -- `blob.service.ssl.enabled`: Flag to enable ssl for the blob client/server communication. This is applicable only when the global ssl flag security.ssl.enabled is set to true (DEFAULT: true). - -- `restart-strategy`: Default [restart strategy]({{site.baseurl}}/dev/restart_strategies.html) to use in case no -restart strategy has been specified for the job. -The options are: - - fixed delay strategy: `fixed-delay`. - - failure rate strategy: `failure-rate`. - - no restarts: `none` - - Default value is `none` unless checkpointing is enabled for the job in which case the default is `fixed-delay` with `Integer.MAX_VALUE` restart attempts and `10s` delay. - -- `restart-strategy.fixed-delay.attempts`: Number of restart attempts, used if the default restart strategy is set to "fixed-delay". -Default value is 1, unless "fixed-delay" was activated by enabling checkpoints, in which case the default is `Integer.MAX_VALUE`. - -- `restart-strategy.fixed-delay.delay`: Delay between restart attempts, used if the default restart strategy is set to "fixed-delay". (default: `1 s`) - -- `restart-strategy.failure-rate.max-failures-per-interval`: Maximum number of restarts in given time interval before failing a job in "failure-rate" strategy. -Default value is 1. - -- `restart-strategy.failure-rate.failure-rate-interval`: Time interval for measuring failure rate in "failure-rate" strategy. -Default value is `1 minute`. - -- `restart-strategy.failure-rate.delay`: Delay between restart attempts, used if the default restart strategy is set to "failure-rate". -Default value is the `akka.ask.timeout`. - -- `jobstore.cache-size`: The job store cache size in bytes which is used to keep completed jobs in memory (DEFAULT: `52428800` (`50` MB)). - -- `jobstore.expiration-time`: The time in seconds after which a completed job expires and is purged from the job store (DEFAULT: `3600`). +{% include generated/common_section.html %} ## Full Reference @@ -358,6 +154,10 @@ The configuration keys in this section are independent of the used resource mana {% include generated/checkpointing_configuration.html %} +### RocksDB State Backend + +{% include generated/rocks_db_configuration.html %} + ### Queryable State {% include generated/queryable_state_configuration.html %} @@ -366,6 +166,16 @@ The configuration keys in this section are independent of the used resource mana {% include generated/metric_configuration.html %} +### RocksDB Native Metrics +Certain RocksDB native metrics may be forwarded to Flink's metrics reporter. +All native metrics are scoped to operators and then further broken down by column family; values are reported as unsigned longs. + +
    + Note: Enabling native metrics may cause degraded performance and should be set carefully. +
    + +{% include generated/rocks_db_native_metric_configuration.html %} + ### History Server You have to configure `jobmanager.archive.fs.dir` in order to archive terminated jobs and add it to the list of monitored directories via `historyserver.archive.fs.dir` if you want to display them via the HistoryServer's web frontend. @@ -374,15 +184,9 @@ You have to configure `jobmanager.archive.fs.dir` in order to archive terminated {% include generated/history_server_configuration.html %} -## Flip-6 - -- `mode`: Execution mode of Flink. Possible values are `old` and `flip6`. In order to start the Flip-6 components, you have to specify `flip6` (DEFAULT: `old`). - -### Slot Manager (Flip-6) - -The configuration keys in this section are relevant for the SlotManager running in the Flip-6 ResourceManager +## Legacy -{% include generated/slot_manager_configuration.html %} +- `mode`: Execution mode of Flink. Possible values are `legacy` and `new`. In order to start the legacy components, you have to specify `legacy` (DEFAULT: `new`). ## Background @@ -421,10 +225,10 @@ Previously, the number of network buffers was set manually which became a quite network buffers with the following configuration parameters: - `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers (DEFAULT: 0.1), -- `taskmanager.network.memory.min`: Minimum memory size for network buffers in bytes (DEFAULT: 64 MB), -- `taskmanager.network.memory.max`: Maximum memory size for network buffers in bytes (DEFAULT: 1 GB), and +- `taskmanager.network.memory.min`: Minimum memory size for network buffers (DEFAULT: 64MB), +- `taskmanager.network.memory.max`: Maximum memory size for network buffers (DEFAULT: 1GB), and - `taskmanager.memory.segment-size`: Size of memory buffers used by the memory manager and the -network stack in bytes (DEFAULT: 32768 (= 32 KiBytes)). +network stack in bytes (DEFAULT: 32KB). #### Setting the Number of Network Buffers directly @@ -439,9 +243,9 @@ active at the same time. Since the *intra-node-parallelism* is typically the num more than 4 repartitioning or broadcasting channels are rarely active in parallel, it frequently boils down to -``` +{% highlight plain %} #slots-per-TM^2 * #TMs * 4 -``` +{% endhighlight %} Where `#slots per TM` are the [number of slots per TaskManager](#configuring-taskmanager-processing-slots) and `#TMs` are the total number of task managers. diff --git a/docs/ops/deployment/aws.md b/docs/ops/deployment/aws.md index 7ef95e7399c850..ae43c40812bd0f 100644 --- a/docs/ops/deployment/aws.md +++ b/docs/ops/deployment/aws.md @@ -58,9 +58,9 @@ After creating your cluster, you can [connect to the master node](http://docs.aw 1. Go the [Downloads Page]({{ site.download_url }}) and **download a binary version of Flink matching the Hadoop version** of your EMR cluster, e.g. Hadoop 2.7 for EMR releases 4.3.0, 4.4.0, or 4.5.0. 2. Extract the Flink distribution and you are ready to deploy [Flink jobs via YARN](yarn_setup.html) after **setting the Hadoop config directory**: -```bash +{% highlight bash %} HADOOP_CONF_DIR=/etc/hadoop/conf ./bin/flink run -m yarn-cluster -yn 1 examples/streaming/WordCount.jar -``` +{% endhighlight %} {% top %} @@ -70,13 +70,13 @@ HADOOP_CONF_DIR=/etc/hadoop/conf ./bin/flink run -m yarn-cluster -yn 1 examples/ You can use S3 objects like regular files by specifying paths in the following format: -``` +{% highlight plain %} s3:/// -``` +{% endhighlight %} The endpoint can either be a single file or a directory, for example: -```java +{% highlight java %} // Read from S3 bucket env.readTextFile("s3:///"); @@ -85,7 +85,7 @@ stream.writeAsText("s3:///"); // Use S3 as FsStatebackend env.setStateBackend(new FsStateBackend("s3:///")); -``` +{% endhighlight %} Note that these examples are *not* exhaustive and you can use S3 in other places as well, including your [high availability setup](../jobmanager_high_availability.html) or the [RocksDBStateBackend]({{ site.baseurl }}/ops/state/state_backends.html#the-rocksdbstatebackend); everywhere that Flink expects a FileSystem URI. @@ -101,9 +101,14 @@ implementation. Both ways are described below. To use either `flink-s3-fs-hadoop` or `flink-s3-fs-presto`, copy the respective JAR file from the `opt` directory to the `lib` directory of your Flink distribution before starting Flink, e.g. -``` +{% highlight bash %} cp ./opt/flink-s3-fs-presto-{{ site.version }}.jar ./lib/ -``` +{% endhighlight %} + +Both `flink-s3-fs-hadoop` and `flink-s3-fs-presto` register default FileSystem +wrappers for URIs with the `s3://` scheme, `flink-s3-fs-hadoop` also registers +for `s3a://` and `flink-s3-fs-presto` also registers for `s3p://`, so you can +use this to use both at the same time. #### Configure Access Credentials @@ -121,10 +126,10 @@ Access to S3 can be granted via your **access and secret key pair**. Please note You need to configure both `s3.access-key` and `s3.secret-key` in Flink's `flink-conf.yaml`: -``` +{% highlight yaml %} s3.access-key: your-access-key s3.secret-key: your-secret-key -``` +{% endhighlight %} {% top %} @@ -149,7 +154,7 @@ This is the recommended S3 FileSystem implementation to use. It uses Amazon's SD You need to point Flink to a valid Hadoop configuration, which contains the following properties in `core-site.xml`: -```xml +{% highlight xml %} @@ -165,7 +170,7 @@ You need to point Flink to a valid Hadoop configuration, which contains the foll -``` +{% endhighlight %} This registers `S3AFileSystem` as the default FileSystem for URIs with the `s3a://` scheme. @@ -175,12 +180,12 @@ This file system is limited to files up to 5GB in size and it does not work with You need to point Flink to a valid Hadoop configuration, which contains the following property in `core-site.xml`: -```xml +{% highlight xml %} fs.s3.impl org.apache.hadoop.fs.s3native.NativeS3FileSystem -``` +{% endhighlight %} This registers `NativeS3FileSystem` as the default FileSystem for URIs with the `s3://` scheme. @@ -192,9 +197,9 @@ You can specify the [Hadoop configuration](../config.html#hdfs) in various ways the path of the Hadoop configuration directory, for example - by setting the environment variable `HADOOP_CONF_DIR`, or - by setting the `fs.hdfs.hadoopconf` configuration option in `flink-conf.yaml`: -``` +{% highlight yaml %} fs.hdfs.hadoopconf: /path/to/etc/hadoop -``` +{% endhighlight %} This registers `/path/to/etc/hadoop` as Hadoop's configuration directory with Flink. Flink will look for the `core-site.xml` and `hdfs-site.xml` files in the specified directory. @@ -222,7 +227,7 @@ Access to S3 can be granted via your **access and secret key pair**. Please note For `S3AFileSystem` you need to configure both `fs.s3a.access.key` and `fs.s3a.secret.key` in Hadoop's `core-site.xml`: -```xml +{% highlight xml %} fs.s3a.access.key @@ -232,7 +237,7 @@ For `S3AFileSystem` you need to configure both `fs.s3a.access.key` and `fs.s3a.s fs.s3a.secret.key -``` +{% endhighlight %} {% top %} @@ -242,7 +247,7 @@ Access to S3 can be granted via your **access and secret key pair**. But this is For `NativeS3FileSystem` you need to configure both `fs.s3.awsAccessKeyId` and `fs.s3.awsSecretAccessKey` in Hadoop's `core-site.xml`: -```xml +{% highlight xml %} fs.s3.awsAccessKeyId @@ -252,7 +257,7 @@ For `NativeS3FileSystem` you need to configure both `fs.s3.awsAccessKeyId` and ` fs.s3.awsSecretAccessKey -``` +{% endhighlight %} {% top %} @@ -320,7 +325,7 @@ The following sections lists common issues when working with Flink on AWS. If your job submission fails with an Exception message noting that `No file system found with scheme s3` this means that no FileSystem has been configured for S3. Please check out the configuration sections for our [shaded Hadoop/Presto](#shaded-hadooppresto-s3-file-systems-recommended) or [generic Hadoop](#set-s3-filesystem) file systems for details on how to configure this properly. -``` +{% highlight plain %} org.apache.flink.client.program.ProgramInvocationException: The program execution failed: Failed to submit job cd927567a81b62d7da4c18eaa91c3c39 (WordCount Example) [...] Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: @@ -332,7 +337,7 @@ Caused by: java.io.IOException: No file system found with scheme s3, at o.a.f.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:450) at o.a.f.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:57) at o.a.f.runtime.executiongraph.ExecutionJobVertex.(ExecutionJobVertex.java:156) -``` +{% endhighlight %} {% top %} @@ -340,7 +345,7 @@ Caused by: java.io.IOException: No file system found with scheme s3, If you see your job failing with an Exception noting that the `AWS Access Key ID and Secret Access Key must be specified as the username or password`, your access credentials have not been set up properly. Please refer to the access credential section for our [shaded Hadoop/Presto](#configure-access-credentials) or [generic Hadoop](#configure-access-credentials-1) file systems for details on how to configure this. -``` +{% highlight plain %} org.apache.flink.client.program.ProgramInvocationException: The program execution failed: Failed to submit job cd927567a81b62d7da4c18eaa91c3c39 (WordCount Example) [...] Caused by: java.io.IOException: The given file URI (s3:///) points to the @@ -362,7 +367,7 @@ Caused by: java.lang.IllegalArgumentException: AWS Access Key ID and Secret Acce at o.a.h.fs.s3native.$Proxy6.initialize(Unknown Source) at o.a.h.fs.s3native.NativeS3FileSystem.initialize(NativeS3FileSystem.java:330) at o.a.f.runtime.fs.hdfs.HadoopFileSystem.initialize(HadoopFileSystem.java:321) -``` +{% endhighlight %} {% top %} @@ -370,7 +375,7 @@ Caused by: java.lang.IllegalArgumentException: AWS Access Key ID and Secret Acce If you see this Exception, the S3 FileSystem is not part of the class path of Flink. Please refer to [S3 FileSystem dependency section](#provide-s3-filesystem-dependency) for details on how to configure this properly. -``` +{% highlight plain %} Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.s3native.NativeS3FileSystem not found at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2186) at org.apache.flink.runtime.fs.hdfs.HadoopFileSystem.getHadoopWrapperClassNameForFileSystem(HadoopFileSystem.java:460) @@ -389,7 +394,7 @@ Caused by: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.s3native at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2060) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2152) ... 33 more -``` +{% endhighlight %} {% top %} @@ -397,16 +402,16 @@ Caused by: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.s3native If you have configured everything properly, but get a `Bad Request` Exception **and** your S3 bucket is located in region `eu-central-1`, you might be running an S3 client, which does not support [Amazon's signature version 4](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). -``` +{% highlight plain %} [...] Caused by: java.io.IOException: s3:/// : 400 : Bad Request [...] Caused by: org.jets3t.service.impl.rest.HttpException [...] -``` +{% endhighlight %} or -``` +{% highlight plain %} com.amazonaws.services.s3.model.AmazonS3Exception: Status Code: 400, AWS Service: Amazon S3, AWS Request ID: [...], AWS Error Code: null, AWS Error Message: Bad Request, S3 Extended Request ID: [...] -``` +{% endhighlight %} This should not apply to our shaded Hadoop/Presto S3 file systems but can occur for Hadoop-provided S3 file systems. In particular, all Hadoop versions up to 2.7.2 running `NativeS3FileSystem` (which @@ -417,9 +422,9 @@ Except for changing the bucket region, you may also be able to solve this by [requesting signature version 4 for request authentication](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version), e.g. by adding this to Flink's JVM options in `flink-conf.yaml` (see [configuration](../config.html#common-options)): -``` +{% highlight yaml %} env.java.opts: -Dcom.amazonaws.services.s3.enableV4 -``` +{% endhighlight %} {% top %} @@ -427,7 +432,7 @@ env.java.opts: -Dcom.amazonaws.services.s3.enableV4 This Exception is usually caused by skipping the local buffer directory configuration `fs.s3a.buffer.dir` for the `S3AFileSystem`. Please refer to the [S3AFileSystem configuration](#s3afilesystem-recommended) section to see how to configure the `S3AFileSystem` properly. -``` +{% highlight plain %} [...] Caused by: java.lang.NullPointerException at o.a.h.fs.LocalDirAllocator$AllocatorPerContext.confChanged(LocalDirAllocator.java:268) at @@ -442,6 +447,6 @@ o.a.h.fs.FileSystem.create(FileSystem.java:785) at o.a.f.runtime.fs.hdfs.HadoopFileSystem.create(HadoopFileSystem.java:404) at o.a.f.runtime.fs.hdfs.HadoopFileSystem.create(HadoopFileSystem.java:48) at ... 25 more -``` +{% endhighlight %} {% top %} diff --git a/docs/ops/deployment/cluster_setup.md b/docs/ops/deployment/cluster_setup.md index c8a5d0b2845935..bb59f165e27606 100644 --- a/docs/ops/deployment/cluster_setup.md +++ b/docs/ops/deployment/cluster_setup.md @@ -59,10 +59,10 @@ Go to the [downloads page]({{ site.download_url }}) and get the ready-to-run pac After downloading the latest release, copy the archive to your master node and extract it: -~~~bash +{% highlight bash %} tar xzf flink-*.tgz cd flink-* -~~~ +{% endhighlight %} ### Configuring Flink @@ -122,9 +122,9 @@ The following script starts a JobManager on the local node and connects via SSH Assuming that you are on the master node and inside the Flink directory: -~~~bash +{% highlight bash %} bin/start-cluster.sh -~~~ +{% endhighlight %} To stop Flink, there is also a `stop-cluster.sh` script. @@ -136,15 +136,15 @@ You can add both JobManager and TaskManager instances to your running cluster wi #### Adding a JobManager -~~~bash -bin/jobmanager.sh ((start|start-foreground) cluster)|stop|stop-all -~~~ +{% highlight bash %} +bin/jobmanager.sh ((start|start-foreground) [host] [webui-port])|stop|stop-all +{% endhighlight %} #### Adding a TaskManager -~~~bash +{% highlight bash %} bin/taskmanager.sh start|start-foreground|stop|stop-all -~~~ +{% endhighlight %} Make sure to call these scripts on the hosts on which you want to start/stop the respective instance. diff --git a/docs/ops/deployment/docker.md b/docs/ops/deployment/docker.md index 4986f2ae7aa341..baa54868bb6f9f 100644 --- a/docs/ops/deployment/docker.md +++ b/docs/ops/deployment/docker.md @@ -23,53 +23,58 @@ specific language governing permissions and limitations under the License. --> -[Docker](https://www.docker.com) is a popular container runtime. There are -official Docker images for Apache Flink available on Docker Hub which can be -used directly or extended to better integrate into a production environment. +[Docker](https://www.docker.com) is a popular container runtime. +There are Docker images for Apache Flink available on Docker Hub which can be used to deploy a session cluster. +The Flink repository also contains tooling to create container images to deploy a job cluster. * This will be replaced by the TOC {:toc} -## Official Docker Images +## Flink session cluster -The [official Docker repository](https://hub.docker.com/_/flink/) is -hosted on Docker Hub and serves images of Flink version 1.2.1 and later. +A Flink session cluster can be used to run multiple jobs. +Each job needs to be submitted to the cluster after it has been deployed. -Images for each supported combination of Hadoop and Scala are available, and -tag aliases are provided for convenience. +### Docker images -For example, the following aliases can be used: *(`1.2.y` indicates the latest -release of Flink 1.2)* +The [Flink Docker repository](https://hub.docker.com/_/flink/) is hosted on +Docker Hub and serves images of Flink version 1.2.1 and later. -* `flink:latest` → -`flink:-hadoop-scala_` -* `flink:1.2` → `flink:1.2.y-hadoop27-scala_2.11` -* `flink:1.2.1-scala_2.10` → `flink:1.2.1-hadoop27-scala_2.10` -* `flink:1.2-hadoop26` → `flink:1.2.y-hadoop26-scala_2.11` +Images for each supported combination of Hadoop and Scala are available, and tag aliases are provided for convenience. - - +* `flink:latest` → `flink:-scala_` +* `flink:1.5` → `flink:1.5.y-scala_2.11` +* `flink:1.5-hadoop27` → `flink:1.5.y-hadoop27-scala_2.11` -**Note:** The docker images are provided as a community project by individuals +**Note:** The Docker images are provided as a community project by individuals on a best-effort basis. They are not official releases by the Apache Flink PMC. +## Flink job cluster + +A Flink job cluster is a dedicated cluster which runs a single job. +The job is part of the image and, thus, there is no extra job submission needed. + +### Docker images + +The Flink job cluster image needs to contain the user code jars of the job for which the cluster is started. +Therefore, one needs to build a dedicated container image for every job. +The `flink-container` module contains a `build.sh` script which can be used to create such an image. +Please see the [instructions](https://github.com/apache/flink/blob/{{ site.github_branch }}/flink-container/docker/README.md) for more details. + ## Flink with Docker Compose [Docker Compose](https://docs.docker.com/compose/) is a convenient way to run a group of Docker containers locally. -An [example config file](https://github.com/docker-flink/examples/blob/master/docker-compose.yml) -is available on GitHub. +Example config files for a [session cluster](https://github.com/docker-flink/examples/blob/master/docker-compose.yml) and a [job cluster](https://github.com/apache/flink/blob/{{ site.github_branch }}/flink-container/docker/docker-compose.yml) +are available on GitHub. ### Usage @@ -85,10 +90,14 @@ is available on GitHub. docker-compose scale taskmanager= -When the cluster is running, you can visit the web UI at [http://localhost:8081 -](http://localhost:8081) and submit a job. +* Kill the cluster + + docker-compose kill + +When the cluster is running, you can visit the web UI at [http://localhost:8081](http://localhost:8081). +You can also use the web UI to submit a job to a session cluster. -To submit a job via the command line, you must copy the JAR to the Jobmanager +To submit a job to a session cluster via the command line, you must copy the JAR to the JobManager container and submit the job from there. For example: diff --git a/docs/ops/deployment/hadoop.md b/docs/ops/deployment/hadoop.md index cd3c096dcd9950..b07f9259e688d0 100644 --- a/docs/ops/deployment/hadoop.md +++ b/docs/ops/deployment/hadoop.md @@ -40,8 +40,10 @@ running inside YARN will be started with the Hadoop classpaths, but it can happen that the Hadoop dependencies must be in the classpath when submitting a job to YARN. For this, it's usually enough to do a -``` +{% highlight bash %} export HADOOP_CLASSPATH=`hadoop classpath` -``` +{% endhighlight %} in the shell. Note that `hadoop` is the hadoop binary and that `classpath` is an argument that will make it print the configured Hadoop classpath. + +{% top %} diff --git a/docs/ops/deployment/kubernetes.md b/docs/ops/deployment/kubernetes.md index 0790a056918f2a..5244f5ed544b3a 100644 --- a/docs/ops/deployment/kubernetes.md +++ b/docs/ops/deployment/kubernetes.md @@ -23,51 +23,77 @@ specific language governing permissions and limitations under the License. --> -[Kubernetes](https://kubernetes.io) is a container orchestration system. +This page describes how to deploy a Flink job and session cluster on [Kubernetes](https://kubernetes.io). * This will be replaced by the TOC {:toc} -## Simple Kubernetes Flink Cluster +## Setup Kubernetes -A basic Flink cluster deployment in Kubernetes has three components: +Please follow [Kubernetes' setup guide](https://kubernetes.io/docs/setup/) in order to deploy a Kubernetes cluster. +If you want to run Kubernetes locally, we recommend using [MiniKube](https://kubernetes.io/docs/setup/minikube/). -* a Deployment for a single Jobmanager -* a Deployment for a pool of Taskmanagers -* a Service exposing the Jobmanager's RPC and UI ports +
    + Note: If using MiniKube please make sure to execute `minikube ssh 'sudo ip link set docker0 promisc on'` before deploying a Flink cluster. + Otherwise Flink components are not able to self reference themselves through a Kubernetes service. +
    -### Launching the cluster +## Flink session cluster on Kubernetes -Using the [resource definitions found below](#simple-kubernetes-flink-cluster- -resources), launch the cluster with the `kubectl` command: +A Flink session cluster is executed as a long-running Kubernetes Deployment. +Note that you can run multiple Flink jobs on a session cluster. +Each job needs to be submitted to the cluster after the cluster has been deployed. +A basic Flink session cluster deployment in Kubernetes has three components: + +* a Deployment/Job which runs the JobManager +* a Deployment for a pool of TaskManagers +* a Service exposing the JobManager's REST and UI ports + +### Deploy Flink session cluster on Kubernetes + +Using the resource definitions for a [session cluster](#session-cluster-resource-definitions), launch the cluster with the `kubectl` command: + + kubectl create -f jobmanager-service.yaml kubectl create -f jobmanager-deployment.yaml kubectl create -f taskmanager-deployment.yaml - kubectl create -f jobmanager-service.yaml You can then access the Flink UI via `kubectl proxy`: 1. Run `kubectl proxy` in a terminal -2. Navigate to [http://localhost:8001/api/v1/proxy/namespaces/default/services/flink-jobmanager:8081 -](http://localhost:8001/api/v1/proxy/namespaces/default/services/flink- -jobmanager:8081) in your browser - -### Deleting the cluster +2. Navigate to [http://localhost:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy](http://localhost:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy) in your browser -Again, use `kubectl` to delete the cluster: +In order to terminate the Flink session cluster, use `kubectl`: kubectl delete -f jobmanager-deployment.yaml kubectl delete -f taskmanager-deployment.yaml kubectl delete -f jobmanager-service.yaml +## Flink job cluster on Kubernetes + +A Flink job cluster is a dedicated cluster which runs a single job. +The job is part of the image and, thus, there is no extra job submission needed. + +### Creating the job-specific image + +The Flink job cluster image needs to contain the user code jars of the job for which the cluster is started. +Therefore, one needs to build a dedicated container image for every job. +Please follow these [instructions](https://github.com/apache/flink/blob/{{ site.github_branch }}/flink-container/docker/README.md) to build the Docker image. + +### Deploy Flink job cluster on Kubernetes + +In order to deploy the a job cluster on Kubernetes please follow these [instructions](https://github.com/apache/flink/blob/{{ site.github_branch }}/flink-container/kubernetes/README.md#deploy-flink-job-cluster). + ## Advanced Cluster Deployment -An early version of a [Flink Helm chart](https://github.com/docker-flink/ -examples) is available on GitHub. +An early version of a [Flink Helm chart](https://github.com/docker-flink/examples) is available on GitHub. ## Appendix -### Simple Kubernetes Flink cluster resources +### Session cluster resource definitions + +The Deployment definitions use the pre-built image `flink:latest` which can be found [on Docker Hub](https://hub.docker.com/r/_/flink/). +The image is built from this [Github repository](https://github.com/docker-flink/docker-flink). `jobmanager-deployment.yaml` {% highlight yaml %} diff --git a/docs/ops/deployment/mapr_setup.md b/docs/ops/deployment/mapr_setup.md index 19920ad9076b2a..37b192f94b7b8d 100644 --- a/docs/ops/deployment/mapr_setup.md +++ b/docs/ops/deployment/mapr_setup.md @@ -41,11 +41,11 @@ In order to run Flink on MapR, Flink needs to be built with MapR's own Hadoop and Zookeeper distribution. Simply build Flink using Maven with the following command from the project root directory: -``` +{% highlight bash %} mvn clean install -DskipTests -Pvendor-repos,mapr \ -Dhadoop.version=2.7.0-mapr-1607 \ -Dzookeeper.version=3.4.5-mapr-1604 -``` +{% endhighlight %} The `vendor-repos` build profile adds MapR's repository to the build so that MapR's Hadoop / Zookeeper dependencies can be fetched. The `mapr` build @@ -67,13 +67,13 @@ The client submitting Flink jobs to MapR also needs to be prepared with the belo Ensure that MapR's JAAS config file is picked up to avoid login failures: -``` +{% highlight bash %} export JVM_ARGS=-Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -``` +{% endhighlight %} Make sure that the `yarn.nodemanager.resource.cpu-vcores` property is set in `yarn-site.xml`: -~~~xml +{% highlight xml %} @@ -86,28 +86,28 @@ Make sure that the `yarn.nodemanager.resource.cpu-vcores` property is set in `ya ... -~~~ +{% endhighlight %} Also remember to set the `YARN_CONF_DIR` or `HADOOP_CONF_DIR` environment variables to the path where `yarn-site.xml` is located: -``` +{% highlight bash %} export YARN_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/ export HADOOP_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/ -``` +{% endhighlight %} Make sure that the MapR native libraries are picked up in the classpath: -``` +{% highlight bash %} export FLINK_CLASSPATH=/opt/mapr/lib/* -``` +{% endhighlight %} If you'll be starting Flink on YARN sessions with `yarn-session.sh`, the below is also required: -``` +{% highlight bash %} export CC_CLASSPATH=/opt/mapr/lib/* -``` +{% endhighlight %} ## Running Flink with a Secured MapR Cluster @@ -125,10 +125,10 @@ Users simply need to login by using MapR's `maprlogin` authentication utility. Users that haven't acquired MapR login credentials would not be able to submit Flink jobs, erroring with: -``` +{% highlight plain %} java.lang.Exception: unable to establish the security context Caused by: o.a.f.r.security.modules.SecurityModule$SecurityInstallException: Unable to set the Hadoop login user Caused by: java.io.IOException: failure to login: Unable to obtain MapR credentials -``` +{% endhighlight %} {% top %} diff --git a/docs/ops/deployment/mesos.md b/docs/ops/deployment/mesos.md index 56f9d9310cac67..1ff8afad74ebb5 100644 --- a/docs/ops/deployment/mesos.md +++ b/docs/ops/deployment/mesos.md @@ -59,13 +59,11 @@ or configuration files. For instance, in non-containerized environments, the artifact server will provide the Flink binaries. What files will be served depends on the configuration overlay used. -### Flink's JobManager and Web Interface +### Flink's Dispatcher and Web Interface -The Mesos scheduler currently resides with the JobManager but will be started -independently of the JobManager in future versions (see -[FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077)). The -proposed changes will also add a Dipsatcher component which will be the central -point for job submission and monitoring. +The Dispatcher and the web interface provide a central point for monitoring, +job submission, and other client interaction with the cluster +(see [FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077)). ### Startup script and configuration overlays @@ -101,7 +99,7 @@ You can also run Mesos without DC/OS. ### Installing Mesos -Please follow the [instructions on how to setup Mesos on the official website](http://mesos.apache.org/documentation/latest/getting-started/). +Please follow the [instructions on how to setup Mesos on the official website](http://mesos.apache.org/getting-started/). After installation you have to configure the set of master and agent nodes by creating the files `MESOS_HOME/etc/mesos/masters` and `MESOS_HOME/etc/mesos/slaves`. These files contain in each row a single hostname on which the respective component will be started (assuming SSH access to these nodes). @@ -139,7 +137,7 @@ More information about the deployment scripts can be found [here](http://mesos.a ### Installing Marathon -Optionally, you may also [install Marathon](https://mesosphere.github.io/marathon/docs/) which will be necessary to run Flink in high availability (HA) mode. +Optionally, you may also [install Marathon](https://mesosphere.github.io/marathon/docs/) which enables you to run Flink in [high availability (HA) mode](#high-availability). ### Pre-installing Flink vs Docker/Mesos containers @@ -171,8 +169,6 @@ which manage the Flink processes in a Mesos cluster: It is automatically launched by the Mesos worker node to bring up a new TaskManager. In order to run the `mesos-appmaster.sh` script you have to define `mesos.master` in the `flink-conf.yaml` or pass it via `-Dmesos.master=...` to the Java process. -Additionally, you should define the number of task managers which are started by Mesos via `mesos.initial-tasks`. -This value can also be defined in the `flink-conf.yaml` or passed as a Java property. When executing `mesos-appmaster.sh`, it will create a job manager on the machine where you executed the script. In contrast to that, the task managers will be run as Mesos tasks in the Mesos cluster. @@ -187,20 +183,22 @@ For example: -Dmesos.master=master.foobar.org:5050 \ -Djobmanager.heap.mb=1024 \ -Djobmanager.rpc.port=6123 \ - -Djobmanager.web.port=8081 \ - -Dmesos.initial-tasks=10 \ + -Drest.port=8081 \ -Dmesos.resourcemanager.tasks.mem=4096 \ -Dtaskmanager.heap.mb=3500 \ -Dtaskmanager.numberOfTaskSlots=2 \ -Dparallelism.default=10 +
    + Note: If Flink is in legacy mode, + you should additionally define the number of task managers that are started by Mesos via + mesos.initial-tasks. +
    ### High Availability You will need to run a service like Marathon or Apache Aurora which takes care of restarting the Flink master process in case of node or process failures. -In addition, Zookeeper needs to be configured like described in the [High Availability section of the Flink docs]({{ site.baseurl }}/ops/jobmanager_high_availability.html) - -For the reconciliation of tasks to work correctly, please also set `high-availability.zookeeper.path.mesos-workers` to a valid Zookeeper path. +In addition, Zookeeper needs to be configured like described in the [High Availability section of the Flink docs]({{ site.baseurl }}/ops/jobmanager_high_availability.html). #### Marathon @@ -211,7 +209,7 @@ Here is an example configuration for Marathon: { "id": "flink", - "cmd": "$FLINK_HOME/bin/mesos-appmaster.sh -Djobmanager.heap.mb=1024 -Djobmanager.rpc.port=6123 -Djobmanager.web.port=8081 -Dmesos.initial-tasks=1 -Dmesos.resourcemanager.tasks.mem=1024 -Dtaskmanager.heap.mb=1024 -Dtaskmanager.numberOfTaskSlots=2 -Dparallelism.default=2 -Dmesos.resourcemanager.tasks.cpus=1", + "cmd": "$FLINK_HOME/bin/mesos-appmaster.sh -Djobmanager.heap.mb=1024 -Djobmanager.rpc.port=6123 -Drest.port=8081 -Dmesos.resourcemanager.tasks.mem=1024 -Dtaskmanager.heap.mb=1024 -Dtaskmanager.numberOfTaskSlots=2 -Dparallelism.default=2 -Dmesos.resourcemanager.tasks.cpus=1", "cpus": 1.0, "mem": 1024 } @@ -220,56 +218,7 @@ When running Flink with Marathon, the whole Flink cluster including the job mana ### Configuration parameters -`mesos.initial-tasks`: The initial workers to bring up when the master starts (**DEFAULT**: The number of workers specified at cluster startup). - -`mesos.constraints.hard.hostattribute`: Constraints for task placement on Mesos based on agent attributes (**DEFAULT**: None). -Takes a comma-separated list of key:value pairs corresponding to the attributes exposed by the target -mesos agents. Example: `az:eu-west-1a,series:t2` - -`mesos.maximum-failed-tasks`: The maximum number of failed workers before the cluster fails (**DEFAULT**: Number of initial workers). -May be set to -1 to disable this feature. - -`mesos.master`: The Mesos master URL. The value should be in one of the following forms: - -* `host:port` -* `zk://host1:port1,host2:port2,.../path` -* `zk://username:password@host1:port1,host2:port2,.../path` -* `file:///path/to/file` - -`mesos.failover-timeout`: The failover timeout in seconds for the Mesos scheduler, after which running tasks are automatically shut down (**DEFAULT:** 600). - -`mesos.resourcemanager.artifactserver.port`:The config parameter defining the Mesos artifact server port to use. Setting the port to 0 will let the OS choose an available port. - -`mesos.resourcemanager.framework.name`: Mesos framework name (**DEFAULT:** Flink) - -`mesos.resourcemanager.framework.role`: Mesos framework role definition (**DEFAULT:** *) - -`high-availability.zookeeper.path.mesos-workers`: The ZooKeeper root path for persisting the Mesos worker information. - -`mesos.resourcemanager.framework.principal`: Mesos framework principal (**NO DEFAULT**) - -`mesos.resourcemanager.framework.secret`: Mesos framework secret (**NO DEFAULT**) - -`mesos.resourcemanager.framework.user`: Mesos framework user (**DEFAULT:**"") - -`mesos.resourcemanager.artifactserver.ssl.enabled`: Enables SSL for the Flink artifact server (**DEFAULT**: true). Note that `security.ssl.enabled` also needs to be set to `true` encryption to enable encryption. - -`mesos.resourcemanager.tasks.mem`: Memory to assign to the Mesos workers in MB (**DEFAULT**: 1024) - -`mesos.resourcemanager.tasks.cpus`: CPUs to assign to the Mesos workers (**DEFAULT**: 0.0) - -`mesos.resourcemanager.tasks.gpus`: GPUs to assign to the Mesos workers (**DEFAULT**: 0.0) - -`mesos.resourcemanager.tasks.container.type`: Type of the containerization used: "mesos" or "docker" (DEFAULT: mesos); - -`mesos.resourcemanager.tasks.container.image.name`: Image name to use for the container (**NO DEFAULT**) - -`mesos.resourcemanager.tasks.container.volumes`: A comma separated list of `[host_path:]`container_path`[:RO|RW]`. This allows for mounting additional volumes into your container. (**NO DEFAULT**) - -`mesos.resourcemanager.tasks.container.docker.parameters`: Custom parameters to be passed into docker run command when using the docker containerizer. Comma separated list of `key=value` pairs. `value` may contain '=' (**NO DEFAULT**) - -`mesos.resourcemanager.tasks.hostname`: Optional value to define the TaskManager's hostname. The pattern `_TASK_` is replaced by the actual id of the Mesos task. This can be used to configure the TaskManager to use Mesos DNS (e.g. `_TASK_.flink-service.mesos`) for name lookups. (**NO DEFAULT**) - -`mesos.resourcemanager.tasks.bootstrap-cmd`: A command which is executed before the TaskManager is started (**NO DEFAULT**). +For a list of Mesos specific configuration, refer to the [Mesos section]({{ site.baseurl }}/ops/config.html#mesos) +of the configuration documentation. {% top %} diff --git a/docs/ops/deployment/oss.md b/docs/ops/deployment/oss.md new file mode 100644 index 00000000000000..b70676711e54bc --- /dev/null +++ b/docs/ops/deployment/oss.md @@ -0,0 +1,233 @@ +--- +title: "Aliyun Object Storage Service (OSS)" +nav-title: Aliyun OSS +nav-parent_id: deployment +nav-pos: 9 +--- + + +* ToC +{:toc} + + +## OSS: Object Storage Service + +[Aliyun Object Storage Service](https://www.aliyun.com/product/oss) (Aliyun OSS) is widely used especially among China’s cloud users, and it provides cloud object storage for a variety of use cases. + +[Hadoop file system](http://hadoop.apache.org/docs/current/hadoop-aliyun/tools/hadoop-aliyun/index.html) supports OSS since version 2.9.1. Now, you can also use OSS with Fink for **reading** and **writing data**. + +You can access OSS objects like this: + +{% highlight plain %} +oss:/// +{% endhighlight %} + +Below shows how to use OSS with Flink: + +{% highlight java %} +// Read from OSS bucket +env.readTextFile("oss:///"); + +// Write to OSS bucket +dataSet.writeAsText("oss:///") + +{% endhighlight %} + +There are two ways to use OSS with Flink, our shaded `flink-oss-fs-hadoop` will cover most scenarios. However, you may need to set up a specific Hadoop OSS FileSystem implementation if you want use OSS as YARN's resource storage dir ([This patch](https://issues.apache.org/jira/browse/HADOOP-15919) enables YARN to use OSS). Both ways are described below. + +### Shaded Hadoop OSS file system (recommended) + +In order to use `flink-oss-fs-hadoop`, copy the respective JAR file from the opt directory to the lib directory of your Flink distribution before starting Flink, e.g. + +{% highlight bash %} +cp ./opt/flink-oss-fs-hadoop-{{ site.version }}.jar ./lib/ +{% endhighlight %} + +`flink-oss-fs-hadoop` registers default FileSystem wrappers for URIs with the oss:// scheme. + +#### Configurations setup +After setting up the OSS FileSystem wrapper, you need to add some configurations to make sure that Flink is allowed to access your OSS buckets. + +In order to use OSS with Flink more easily, you can use the same configuration keys in `flink-conf.yaml` as in Hadoop's `core-site.xml` + +You can see the configuration keys in the [Hadoop OSS documentation](http://hadoop.apache.org/docs/current/hadoop-aliyun/tools/hadoop-aliyun/index.html). + +There are some required configurations that must be added to `flink-conf.yaml` (**Other configurations defined in Hadoop OSS documentation are advanced configurations which used by performance tuning**): + +{% highlight yaml %} +fs.oss.endpoint: Aliyun OSS endpoint to connect to +fs.oss.accessKeyId: Aliyun access key ID +fs.oss.accessKeySecret: Aliyun access key secret +{% endhighlight %} + +### Hadoop-provided OSS file system - manual setup +This setup is a bit more complex and we recommend using our shaded Hadoop file systems instead (see above) unless required otherwise, e.g. for using OSS as YARN’s resource storage dir via the fs.defaultFS configuration property in Hadoop’s core-site.xml. + +#### Set OSS FileSystem +You need to point Flink to a valid Hadoop configuration, which contains the following properties in core-site.xml: + +{% highlight xml %} + + + + fs.oss.impl + org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem + + + + fs.oss.endpoint + Aliyun OSS endpoint to connect to + Aliyun OSS endpoint to connect to. An up-to-date list is provided in the Aliyun OSS Documentation. + + + + fs.oss.accessKeyId + Aliyun access key ID + + + + fs.oss.accessKeySecret + Aliyun access key secret + + + + fs.oss.buffer.dir + /tmp/oss + + + + + +{% endhighlight %} + +#### Hadoop Configuration + +You can specify the [Hadoop configuration](../config.html#hdfs) in various ways pointing Flink to +the path of the Hadoop configuration directory, for example +- by setting the environment variable `HADOOP_CONF_DIR`, or +- by setting the `fs.hdfs.hadoopconf` configuration option in `flink-conf.yaml`: +{% highlight yaml %} +fs.hdfs.hadoopconf: /path/to/etc/hadoop +{% endhighlight %} + +This registers `/path/to/etc/hadoop` as Hadoop's configuration directory with Flink. Flink will look for the `core-site.xml` and `hdfs-site.xml` files in the specified directory. + +#### Provide OSS FileSystem Dependency + +You can find Hadoop OSS FileSystem are packaged in the hadoop-aliyun artifact. This JAR and all its dependencies need to be added to Flink’s classpath, i.e. the class path of both Job and TaskManagers. + +There are multiple ways of adding JARs to Flink’s class path, the easiest being simply to drop the JARs in Flink’s lib folder. You need to copy the hadoop-aliyun JAR with all its dependencies (You can find these as part of the Hadoop binaries in hadoop-3/share/hadoop/tools/lib). You can also export the directory containing these JARs as part of the HADOOP_CLASSPATH environment variable on all machines. + +## An Example +Below is an example shows the result of our setup (data is generated by TPC-DS tool) + +{% highlight java %} +// Read from OSS bucket +scala> val dataSet = benv.readTextFile("oss:///50/call_center/data-m-00049") +dataSet: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@31940704 + +scala> dataSet.print() +1|AAAAAAAABAAAAAAA|1998-01-01|||2450952|NY Metro|large|2935|1670015|8AM-4PM|Bob Belcher|6|More than other authori|Shared others could not count fully dollars. New members ca|Julius Tran|3|pri|6|cally|730|Ash Hill|Boulevard|Suite 0|Oak Grove|Williamson County|TN|38370|United States|-5|0.11| +2|AAAAAAAACAAAAAAA|1998-01-01|2000-12-31||2450806|Mid Atlantic|medium|1574|594972|8AM-8AM|Felipe Perkins|2|A bit narrow forms matter animals. Consist|Largely blank years put substantially deaf, new others. Question|Julius Durham|5|anti|1|ought|984|Center Hill|Way|Suite 70|Midway|Williamson County|TN|31904|United States|-5|0.12| +3|AAAAAAAACAAAAAAA|2001-01-01|||2450806|Mid Atlantic|medium|1574|1084486|8AM-4PM|Mark Hightower|2|Wrong troops shall work sometimes in a opti|Largely blank years put substantially deaf, new others. Question|Julius Durham|1|ought|2|able|984|Center Hill|Way|Suite 70|Midway|Williamson County|TN|31904|United States|-5|0.01| +4|AAAAAAAAEAAAAAAA|1998-01-01|2000-01-01||2451063|North Midwest|medium|10137|6578913|8AM-4PM|Larry Mccray|2|Dealers make most historical, direct students|Rich groups catch longer other fears; future,|Matthew Clifton|4|ese|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.05| +5|AAAAAAAAEAAAAAAA|2000-01-02|2001-12-31||2451063|North Midwest|small|17398|4610470|8AM-8AM|Larry Mccray|2|Dealers make most historical, direct students|Blue, due beds come. Politicians would not make far thoughts. Specifically new horses partic|Gary Colburn|4|ese|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.12| +6|AAAAAAAAEAAAAAAA|2002-01-01|||2451063|North Midwest|medium|13118|6585236|8AM-4PM|Larry Mccray|5|Silly particles could pro|Blue, due beds come. Politicians would not make far thoughts. Specifically new horses partic|Gary Colburn|5|anti|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.11| +7|AAAAAAAAHAAAAAAA|1998-01-01|||2451024|Pacific Northwest|small|6280|1739560|8AM-4PM|Alden Snyder|6|Major, formal states can suppor|Reduced, subsequent bases could not lik|Frederick Weaver|5|anti|4|ese|415|Jefferson Tenth|Court|Suite 180|Riverside|Walker County|AL|39231|United States|-6|0.00| +8|AAAAAAAAIAAAAAAA|1998-01-01|2000-12-31||2450808|California|small|4766|2459256|8AM-12AM|Wayne Ray|6|Here possible notions arrive only. Ar|Common, free creditors should exper|Daniel Weller|5|anti|2|able|550|Cedar Elm|Ct.|Suite I|Fairview|Williamson County|TN|35709|United States|-5|0.06| + +scala> dataSet.count() +res0: Long = 8 + +// Write to OSS bucket +scala> dataSet.writeAsText("oss:///50/call_center/data-m-00049.1") + +scala> benv.execute("My batch program") +res1: org.apache.flink.api.common.JobExecutionResult = org.apache.flink.api.common.JobExecutionResult@77476fcf + +scala> val newDataSet = benv.readTextFile("oss:///50/call_center/data-m-00049.1") +newDataSet: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@40b70f31 + +scala> newDataSet.count() +res2: Long = 8 + +{% endhighlight %} + +## Common Issues +### Could not find OSS file system +If your job submission fails with an Exception message like below, please check if our shaded jar (flink-oss-fs-hadoop-{{ site.version }}.jar) is in the lib dir. + +{% highlight plain %} +Caused by: org.apache.flink.runtime.client.JobExecutionException: Could not set up JobManager + at org.apache.flink.runtime.jobmaster.JobManagerRunner.(JobManagerRunner.java:176) + at org.apache.flink.runtime.dispatcher.Dispatcher$DefaultJobManagerRunnerFactory.createJobManagerRunner(Dispatcher.java:1058) + at org.apache.flink.runtime.dispatcher.Dispatcher.lambda$createJobManagerRunner$5(Dispatcher.java:308) + at org.apache.flink.util.function.CheckedSupplier.lambda$unchecked$0(CheckedSupplier.java:34) + ... 7 more +Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: Could not find a file system implementation for scheme 'oss'. The scheme is not directly supported by Flink and no Hadoop file system to support this scheme could be loaded. + at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.(ExecutionJobVertex.java:273) + at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:827) + at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:232) + at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:100) + at org.apache.flink.runtime.jobmaster.JobMaster.createExecutionGraph(JobMaster.java:1151) + at org.apache.flink.runtime.jobmaster.JobMaster.createAndRestoreExecutionGraph(JobMaster.java:1131) + at org.apache.flink.runtime.jobmaster.JobMaster.(JobMaster.java:294) + at org.apache.flink.runtime.jobmaster.JobManagerRunner.(JobManagerRunner.java:157) + ... 10 more +Caused by: org.apache.flink.core.fs.UnsupportedFileSystemSchemeException: Could not find a file system implementation for scheme 'oss'. The scheme is not directly supported by Flink and no Hadoop file system to support this scheme could be loaded. + at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:403) + at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:318) + at org.apache.flink.core.fs.Path.getFileSystem(Path.java:298) + at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:587) + at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62) + at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.(ExecutionJobVertex.java:259) + ... 17 more +Caused by: org.apache.flink.core.fs.UnsupportedFileSystemSchemeException: Hadoop is not in the classpath/dependencies. + at org.apache.flink.core.fs.UnsupportedSchemeFactory.create(UnsupportedSchemeFactory.java:64) + at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:399) + ... 22 more +{% endhighlight %} + +### Missing configuration(s) +If your job submission fails with an Exception message like below, please check if the corresponding configurations exits in `flink-conf.yaml` + +{% highlight plain %} +Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: Aliyun OSS endpoint should not be null or empty. Please set proper endpoint with 'fs.oss.endpoint'. + at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.(ExecutionJobVertex.java:273) + at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:827) + at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:232) + at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:100) + at org.apache.flink.runtime.jobmaster.JobMaster.createExecutionGraph(JobMaster.java:1151) + at org.apache.flink.runtime.jobmaster.JobMaster.createAndRestoreExecutionGraph(JobMaster.java:1131) + at org.apache.flink.runtime.jobmaster.JobMaster.(JobMaster.java:294) + at org.apache.flink.runtime.jobmaster.JobManagerRunner.(JobManagerRunner.java:157) + ... 10 more +Caused by: java.lang.IllegalArgumentException: Aliyun OSS endpoint should not be null or empty. Please set proper endpoint with 'fs.oss.endpoint'. + at org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:145) + at org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) + at org.apache.flink.fs.osshadoop.OSSFileSystemFactory.create(OSSFileSystemFactory.java:87) + at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:395) + at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:318) + at org.apache.flink.core.fs.Path.getFileSystem(Path.java:298) + at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:587) + at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62) + at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.(ExecutionJobVertex.java:259) + ... 17 more +{% endhighlight %} diff --git a/docs/ops/deployment/yarn_setup.md b/docs/ops/deployment/yarn_setup.md index 0fb5bf6584d859..3d13e2db9b3718 100644 --- a/docs/ops/deployment/yarn_setup.md +++ b/docs/ops/deployment/yarn_setup.md @@ -32,14 +32,14 @@ under the License. Start a YARN session with 4 Task Managers (each with 4 GB of Heapspace): -~~~bash +{% highlight bash %} # get the hadoop2 package from the Flink download page at # {{ site.download_url }} curl -O tar xvzf flink-{{ site.version }}-bin-hadoop2.tgz cd flink-{{ site.version }}/ -./bin/yarn-session.sh -n 4 -jm 1024 -tm 4096 -~~~ +./bin/yarn-session.sh -n 4 -jm 1024m -tm 4096m +{% endhighlight %} Specify the `-s` flag for the number of processing slots per Task Manager. We recommend to set the number of slots to the number of processors per machine. @@ -47,14 +47,14 @@ Once the session has been started, you can submit jobs to the cluster using the ### Run a Flink job on YARN -~~~bash +{% highlight bash %} # get the hadoop2 package from the Flink download page at # {{ site.download_url }} curl -O tar xvzf flink-{{ site.version }}-bin-hadoop2.tgz cd flink-{{ site.version }}/ -./bin/flink run -m yarn-cluster -yn 4 -yjm 1024 -ytm 4096 ./examples/batch/WordCount.jar -~~~ +./bin/flink run -m yarn-cluster -yn 4 -yjm 1024m -ytm 4096m ./examples/batch/WordCount.jar +{% endhighlight %} ## Flink YARN Session @@ -79,44 +79,44 @@ Download a Flink package for Hadoop >= 2 from the [download page]({{ site.downlo Extract the package using: -~~~bash +{% highlight bash %} tar xvzf flink-{{ site.version }}-bin-hadoop2.tgz cd flink-{{site.version }}/ -~~~ +{% endhighlight %} #### Start a Session Use the following command to start a session -~~~bash +{% highlight bash %} ./bin/yarn-session.sh -~~~ +{% endhighlight %} This command will show you the following overview: -~~~bash +{% highlight bash %} Usage: Required -n,--container Number of YARN container to allocate (=Number of Task Managers) Optional -D Dynamic properties -d,--detached Start detached - -jm,--jobManagerMemory Memory for JobManager Container [in MB] + -jm,--jobManagerMemory Memory for JobManager Container with optional unit (default: MB) -nm,--name Set a custom name for the application on YARN -q,--query Display available YARN resources (memory, cores) -qu,--queue Specify YARN queue. -s,--slots Number of slots per TaskManager - -tm,--taskManagerMemory Memory per TaskManager Container [in MB] + -tm,--taskManagerMemory Memory per TaskManager Container with optional unit (default: MB) -z,--zookeeperNamespace Namespace to create the Zookeeper sub-paths for HA mode -~~~ +{% endhighlight %} Please note that the Client requires the `YARN_CONF_DIR` or `HADOOP_CONF_DIR` environment variable to be set to read the YARN and HDFS configuration. **Example:** Issue the following command to allocate 10 Task Managers, with 8 GB of memory and 32 processing slots each: -~~~bash +{% highlight bash %} ./bin/yarn-session.sh -n 10 -tm 8192 -s 32 -~~~ +{% endhighlight %} The system will use the configuration in `conf/flink-conf.yaml`. Please follow our [configuration guide]({{ site.baseurl }}/ops/config.html) if you want to change something. @@ -131,8 +131,8 @@ Once Flink is deployed in your YARN cluster, it will show you the connection det Stop the YARN session by stopping the unix process (using CTRL+C) or by entering 'stop' into the client. Flink on YARN will only start all requested containers if enough resources are available on the cluster. Most YARN schedulers account for the requested memory of the containers, -some account also for the number of vcores. By default, the number of vcores is equal to the processing slots (`-s`) argument. The `yarn.containers.vcores` allows overwriting the -number of vcores with a custom value. +some account also for the number of vcores. By default, the number of vcores is equal to the processing slots (`-s`) argument. The [`yarn.containers.vcores`]({{ site.baseurl }}/ops/config.html#yarn-containers-vcores) allows overwriting the +number of vcores with a custom value. In order for this parameter to work you should enable CPU scheduling in your cluster. #### Detached YARN Session @@ -148,25 +148,25 @@ Use the YARN utilities (`yarn application -kill `) to stop the YARN sessi Use the following command to start a session -~~~bash +{% highlight bash %} ./bin/yarn-session.sh -~~~ +{% endhighlight %} This command will show you the following overview: -~~~bash +{% highlight bash %} Usage: Required -id,--applicationId YARN application Id -~~~ +{% endhighlight %} As already mentioned, `YARN_CONF_DIR` or `HADOOP_CONF_DIR` environment variable must be set to read the YARN and HDFS configuration. **Example:** Issue the following command to attach to running Flink YARN session `application_1463870264508_0029`: -~~~bash +{% highlight bash %} ./bin/yarn-session.sh -id application_1463870264508_0029 -~~~ +{% endhighlight %} Attaching to a running session uses YARN ResourceManager to determine Job Manager RPC port. @@ -176,15 +176,15 @@ Stop the YARN session by stopping the unix process (using CTRL+C) or by entering Use the following command to submit a Flink program to the YARN cluster: -~~~bash +{% highlight bash %} ./bin/flink -~~~ +{% endhighlight %} Please refer to the documentation of the [command-line client]({{ site.baseurl }}/ops/cli.html). The command will show you a help menu like this: -~~~bash +{% highlight bash %} [...] Action "run" compiles and runs a program. @@ -202,25 +202,25 @@ Action "run" compiles and runs a program. program. Optional flag to override the default value specified in the configuration -~~~ +{% endhighlight %} Use the *run* action to submit a job to YARN. The client is able to determine the address of the JobManager. In the rare event of a problem, you can also pass the JobManager address using the `-m` argument. The JobManager address is visible in the YARN console. **Example** -~~~bash +{% highlight bash %} wget -O LICENSE-2.0.txt http://www.apache.org/licenses/LICENSE-2.0.txt hadoop fs -copyFromLocal LICENSE-2.0.txt hdfs:/// ... ./bin/flink run ./examples/batch/WordCount.jar \ hdfs:///..../LICENSE-2.0.txt hdfs:///.../wordcount-result.txt -~~~ +{% endhighlight %} If there is the following error, make sure that all TaskManagers started: -~~~bash +{% highlight bash %} Exception in thread "main" org.apache.flink.compiler.CompilerException: Available instances could not be determined from job manager: Connection timed out. -~~~ +{% endhighlight %} You can check the number of TaskManagers in the JobManager web interface. The address of this interface is printed in the YARN session console. @@ -235,9 +235,9 @@ Please note that the client then expects the `-yn` value to be set (number of Ta ***Example:*** -~~~bash +{% highlight bash %} ./bin/flink run -m yarn-cluster -yn 2 ./examples/batch/WordCount.jar -~~~ +{% endhighlight %} The command line options of the YARN session are also available with the `./bin/flink` tool. They are prefixed with a `y` or `yarn` (for the long argument options). @@ -275,9 +275,9 @@ In cases where the Flink YARN session fails during the deployment itself, users To enable it, users have to set the `yarn.log-aggregation-enable` property to `true` in the `yarn-site.xml` file. Once that is enabled, users can use the following command to retrieve all log files of a (failed) YARN session. -~~~ +{% highlight bash %} yarn logs -applicationId -~~~ +{% endhighlight %} Note that it takes a few seconds after the session has finished until the logs show up. @@ -291,7 +291,7 @@ It allows to access log files for running YARN applications and shows diagnostic ## Build YARN client for a specific Hadoop version -Users using Hadoop distributions from companies like Hortonworks, Cloudera or MapR might have to build Flink against their specific versions of Hadoop (HDFS) and YARN. Please read the [build instructions]({{ site.baseurl }}/start/building.html) for more details. +Users using Hadoop distributions from companies like Hortonworks, Cloudera or MapR might have to build Flink against their specific versions of Hadoop (HDFS) and YARN. Please read the [build instructions]({{ site.baseurl }}/flinkDev/building.html) for more details. ## Running Flink on YARN behind Firewalls @@ -324,9 +324,9 @@ This section briefly describes how Flink and YARN interact. -The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and to HDFS. It determines the Hadoop configuration using the following strategy: +The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and HDFS. It determines the Hadoop configuration using the following strategy: -* Test if `YARN_CONF_DIR`, `HADOOP_CONF_DIR` or `HADOOP_CONF_PATH` are set (in that order). If one of these variables are set, they are used to read the configuration. +* Test if `YARN_CONF_DIR`, `HADOOP_CONF_DIR` or `HADOOP_CONF_PATH` are set (in that order). If one of these variables is set, it is used to read the configuration. * If the above strategy fails (this should not be the case in a correct YARN setup), the client is using the `HADOOP_HOME` environment variable. If it is set, the client tries to access `$HADOOP_HOME/etc/hadoop` (Hadoop 2) and `$HADOOP_HOME/conf` (Hadoop 1). When starting a new Flink YARN session, the client first checks if the requested resources (containers and memory) are available. After that, it uploads a jar that contains Flink and the configuration to HDFS (step 1). diff --git a/docs/ops/filesystems.md b/docs/ops/filesystems.md index 75ddf6d156af7a..77757b00dcf7f8 100644 --- a/docs/ops/filesystems.md +++ b/docs/ops/filesystems.md @@ -22,34 +22,53 @@ specific language governing permissions and limitations under the License. --> -This page provides details on setting up and configuring distributed file systems for use with Flink. +This page provides details on setting up and configuring different file systems for use with Flink. +We start by describing how to use and configure the different file systems that are supported by Flink +out-of-the-box, before describing the necessary steps in order to add support about other/custom file system +implementations. -## Flink' File System support +## Flink's File System support -Flink uses file systems both as a source and sink in streaming/batch applications, and as a target for checkpointing. +Flink uses file systems both as *sources* and *sinks* in streaming/batch applications and as a target for *checkpointing*. These file systems can for example be *Unix/Windows file systems*, *HDFS*, or even object stores like *S3*. The file system used for a specific file is determined by the file URI's scheme. For example `file:///home/user/text.txt` refers to a file in the local file system, while `hdfs://namenode:50010/data/user/text.txt` refers to a file in a specific HDFS cluster. File systems are represented via the `org.apache.flink.core.fs.FileSystem` class, which captures the ways to access and modify -files and objects in that file system. FileSystem instances are instantiates once per process and then cached / pooled, to -avoid configuration overhead per stream creation, and to enforce certain constraints, like connection/stream limits. +files and objects in that file system. FileSystem instances are instantiated once per process and then cached / pooled, to +avoid configuration overhead per stream creation and to enforce certain constraints, such as connection/stream limits. ### Built-in File Systems -Flink directly implements the following file systems: +Flink ships with support for most of the popular file systems, namely *local*, *hadoop-compatible*, *S3*, *MapR FS* +and *OpenStack Swift FS*. Each of these is identified by the scheme included in the URI of the provide file path. + +Flink ships with implementations for the following file systems: - **local**: This file system is used when the scheme is *"file://"*, and it represents the file system of the local machine, including any NFS or SAN that is mounted into that local file system. - - **S3**: Flink directly provides file systems to talk to Amazon S3, registered under the scheme *"s3://"*. -There are two alternative implementations, `flink-s3-fs-presto` and `flink-s3-fs-hadoop`, based on code from the [Presto project](https://prestodb.io/) -and the [Hadoop Project](https://hadoop.apache.org/). Both implementations are self-contained with no dependency footprint. -To use those when using Flink as a library, add the respective maven dependency (`org.apache.flink:flink-s3-fs-presto:{{ site.version }}` or `org.apache.flink:flink-s3-fs-hadoop:{{ site.version }}`). -When starting a Flink application from the Flink binaries, copy or move the respective jar file from the `opt` folder to the `lib` folder. -See [AWS setup](deployment/aws.html) for details. - + - **S3**: Flink directly provides file systems to talk to Amazon S3. There are two alternative implementations, `flink-s3-fs-presto` + and `flink-s3-fs-hadoop`. Both implementations are self-contained with no dependency footprint. There is no need to add Hadoop to + the classpath to use them. Both internally use some Hadoop code, but "shade away" all classes to avoid any dependency conflicts. + + - `flink-s3-fs-presto`, registered under the scheme *"s3://"* and *"s3p://"*, is based on code from the [Presto project](https://prestodb.io/). + You can configure it the same way you can [configure the Presto file system](https://prestodb.io/docs/0.185/connector/hive.html#amazon-s3-configuration). + + - `flink-s3-fs-hadoop`, registered under *"s3://"* and *"s3a://"*, based on code from the [Hadoop Project](https://hadoop.apache.org/). + The file system can be [configured exactly like Hadoop's s3a](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html#S3A). + + To use those file systems when using Flink as a library, add the respective maven dependency (`org.apache.flink:flink-s3-fs-presto:{{ site.version }}` + or `org.apache.flink:flink-s3-fs-hadoop:{{ site.version }}`). When starting a Flink application from the Flink binaries, copy or move + the respective jar file from the `opt` folder to the `lib` folder. See also [AWS setup](deployment/aws.html) for additional details. + + Attention: As described above, both Hadoop and Presto "listen" to paths with scheme set to *"s3://"*. This is + convenient for switching between implementations (Hadoop or Presto), but it can lead to non-determinism when both + implementations are required. This can happen when, for example, the job uses the [StreamingFileSink]({{ site.baseurl}}/dev/connectors/streamfile_sink.html) + which only supports Hadoop, but uses Presto for checkpointing. In this case, it is advised to use explicitly *"s3a://"* + as a scheme for the sink (Hadoop) and *"s3p://"* for checkpointing (Presto). + - **MapR FS**: The MapR file system *"maprfs://"* is automatically available when the MapR libraries are in the classpath. - **OpenStack Swift FS**: Flink directly provides a file system to talk to the OpenStack Swift file system, registered under the scheme *"swift://"*. @@ -57,10 +76,10 @@ See [AWS setup](deployment/aws.html) for details. To use it when using Flink as a library, add the respective maven dependency (`org.apache.flink:flink-swift-fs-hadoop:{{ site.version }}` When starting a Flink application from the Flink binaries, copy or move the respective jar file from the `opt` folder to the `lib` folder. -### HDFS and Hadoop File System support +#### HDFS and Hadoop File System support -For a scheme where Flink does not implemented a file system itself, Flink will try to use Hadoop to instantiate a file system for the respective scheme. -All Hadoop file systems are automatically available once `flink-runtime` and the relevant Hadoop libraries are in classpath. +For all schemes where it cannot find a directly supported file system, Flink will try to use Hadoop to instantiate a file system for the respective scheme. +All Hadoop file systems are automatically available once `flink-runtime` and the Hadoop libraries are in classpath. That way, Flink seamlessly supports all of Hadoop file systems, and all Hadoop-compatible file systems (HCFS), for example: @@ -73,18 +92,18 @@ That way, Flink seamlessly supports all of Hadoop file systems, and all Hadoop-c ## Common File System configurations -The following configuration settings exist across different file systems +The following configuration settings exist across different file systems. #### Default File System If paths to files do not explicitly specify a file system scheme (and authority), a default scheme (and authority) will be used. -~~~ +{% highlight yaml %} fs.default-scheme: -~~~ +{% endhighlight %} -For example, if the default file system configured as `fs.default-scheme: hdfs://localhost:9000/`, then a a file path of -`/user/hugo/in.txt'` is interpreted as `hdfs://localhost:9000/user/hugo/in.txt'` +For example, if the default file system configured as `fs.default-scheme: hdfs://localhost:9000/`, then a file path of +`/user/hugo/in.txt` is interpreted as `hdfs://localhost:9000/user/hugo/in.txt`. #### Connection limiting @@ -96,13 +115,13 @@ For example, very small HDFS clusters with few RPC handlers can sometimes be ove To limit a specific file system's connections, add the following entries to the Flink configuration. The file system to be limited is identified by its scheme. -~~~ +{% highlight yaml %} fs..limit.total: (number, 0/-1 mean no limit) fs..limit.input: (number, 0/-1 mean no limit) fs..limit.output: (number, 0/-1 mean no limit) fs..limit.timeout: (milliseconds, 0 means infinite) fs..limit.stream-timeout: (milliseconds, 0 means infinite) -~~~ +{% endhighlight %} You can limit the number if input/output connections (streams) separately (`fs..limit.input` and `fs..limit.output`), as well as impose a limit on the total number of concurrent streams (`fs..limit.total`). If the file system tries to open more streams, the operation will block until some streams are closed. @@ -112,9 +131,36 @@ To prevent inactive streams from taking up the complete pool (preventing new con `fs..limit.stream-timeout`. If a stream does not read/write any bytes for at least that amount of time, it is forcibly closed. These limits are enforced per TaskManager, so each TaskManager in a Flink application or cluster will open up to that number of connections. -In addition, the The limit are also enforced only per FileSystem instance. Because File Systems are created per scheme and authority, different +In addition, the limits are also only enforced per FileSystem instance. Because File Systems are created per scheme and authority, different authorities will have their own connection pool. For example `hdfs://myhdfs:50010/` and `hdfs://anotherhdfs:4399/` will have separate pools. +## Entropy injection for S3 file systems + +The bundled S3 file systems (`flink-s3-fs-presto` and `flink-s3-fs-hadoop`) support entropy injection. Entropy injection is +a technique to improve scalability of AWS S3 buckets through adding some random characters near the beginning of the key. + +If entropy injection is activated, a configured substring in the paths will be replaced by random characters. For example, path +`s3://my-bucket/checkpoints/_entropy_/dashboard-job/` would be replaced by something like `s3://my-bucket/checkpoints/gf36ikvg/dashboard-job/`. + +**Note that this only happens when the file creation passes the option to inject entropy!**, otherwise the file path will +simply remove the entropy key substring. See +[FileSystem.create(Path, WriteOption)](https://ci.apache.org/projects/flink/flink-docs-release-1.6/api/java/org/apache/flink/core/fs/FileSystem.html#create-org.apache.flink.core.fs.Path-org.apache.flink.core.fs.FileSystem.WriteOptions-) +for details. + +*Note: The Flink runtime currently passes the option to inject entropy only to checkpoint data files.* +*All other files, including checkpoint metadata and external URI do not inject entropy, to keep checkpoint URIs predictable.* + +To enable entropy injection, configure the *entropy key* and the *entropy length* parameters. + +``` +s3.entropy.key: _entropy_ +s3.entropy.length: 4 (default) + +``` + +The `s3.entropy.key` defines the string in paths that is replaced by the random characters. Paths that do not contain the entropy key are left unchanged. +If a file system operation does not pass the *"inject entropy"* write option, the entropy key substring is simply removed. +The `s3.entropy.length` defined the number of random alphanumeric characters to replace the entropy key with. ## Adding new File System Implementations diff --git a/docs/ops/jobmanager_high_availability.md b/docs/ops/jobmanager_high_availability.md index a99455849c418e..82297477435231 100644 --- a/docs/ops/jobmanager_high_availability.md +++ b/docs/ops/jobmanager_high_availability.md @@ -44,7 +44,7 @@ As an example, consider the following setup with three JobManager instances: To enable JobManager High Availability you have to set the **high-availability mode** to *zookeeper*, configure a **ZooKeeper quorum** and set up a **masters file** with all JobManagers hosts and their web UI ports. -Flink leverages **[ZooKeeper](http://zookeeper.apache.org)** for *distributed coordination* between all running JobManager instances. ZooKeeper is a separate service from Flink, which provides highly reliable distributed coordination via leader election and light-weight consistent state storage. Check out [ZooKeeper's Getting Started Guide](http://zookeeper.apache.org/doc/trunk/zookeeperStarted.html) for more information about ZooKeeper. Flink includes scripts to [bootstrap a simple ZooKeeper](#bootstrap-zookeeper) installation. +Flink leverages **[ZooKeeper](http://zookeeper.apache.org)** for *distributed coordination* between all running JobManager instances. ZooKeeper is a separate service from Flink, which provides highly reliable distributed coordination via leader election and light-weight consistent state storage. Check out [ZooKeeper's Getting Started Guide](http://zookeeper.apache.org/doc/current/zookeeperStarted.html) for more information about ZooKeeper. Flink includes scripts to [bootstrap a simple ZooKeeper](#bootstrap-zookeeper) installation. #### Masters File (masters) @@ -65,6 +65,7 @@ By default, the job manager will pick a *random port* for inter process communic In order to start an HA-cluster add the following configuration keys to `conf/flink-conf.yaml`: - **high-availability mode** (required): The *high-availability mode* has to be set in `conf/flink-conf.yaml` to *zookeeper* in order to enable high availability mode. +Alternatively this option can be set to FQN of factory class Flink should use to create HighAvailabilityServices instance. 
    high-availability: zookeeper
    @@ -93,7 +94,7 @@ In order to start an HA-cluster add the following configuration keys to `conf/fl - **Storage directory** (required): JobManager metadata is persisted in the file system *storageDir* and only a pointer to this state is stored in ZooKeeper. 
    -high-availability.zookeeper.storageDir: hdfs:///flink/recovery
+high-availability.storageDir: hdfs:///flink/recovery
    The `storageDir` stores all metadata needed to recover a JobManager failure. @@ -109,7 +110,7 @@ high-availability: zookeeper high-availability.zookeeper.quorum: localhost:2181 high-availability.zookeeper.path.root: /flink high-availability.cluster-id: /cluster_one # important: customize per cluster -high-availability.zookeeper.storageDir: hdfs:///flink/recovery +high-availability.storageDir: hdfs:///flink/recovery 2. **Configure masters** in `conf/masters`: @@ -191,7 +192,7 @@ This means that the application can be restarted 9 times for failed attempts bef 
     high-availability: zookeeper
 high-availability.zookeeper.quorum: localhost:2181
-high-availability.zookeeper.storageDir: hdfs:///flink/recovery
+high-availability.storageDir: hdfs:///flink/recovery
 high-availability.zookeeper.path.root: /flink
 yarn.application-attempts: 10
    diff --git a/docs/ops/production_ready.md b/docs/ops/production_ready.md index 0d11b8a1866b11..08fef2bb10d66c 100644 --- a/docs/ops/production_ready.md +++ b/docs/ops/production_ready.md @@ -87,4 +87,14 @@ you are sure that your state will never exceed main memory and blocking the stre you **could consider** to not use the RocksDB backends. However, at this point, we **strongly recommend** using RocksDB for production. +### Config JobManager High Availability(HA) + +The JobManager coordinates every Flink deployment. It is responsible for both *scheduling* and *resource management*. + +By default, there is a single JobManager instance per Flink cluster. This creates a *single point of failure* (SPOF): +if the JobManager crashes, no new programs can be submitted and running programs fail. + +With JobManager High Availability, you can recover from JobManager failures and thereby eliminate the *SPOF*. +We **strongly recommend** you configure [high availability]({{ site.baseurl }}/ops/jobmanager_high_availability.html) for production. + {% top %} diff --git a/docs/ops/scala_shell.md b/docs/ops/scala_shell.md new file mode 100644 index 00000000000000..50987af2ae9e5a --- /dev/null +++ b/docs/ops/scala_shell.md @@ -0,0 +1,303 @@ +--- +title: "Scala REPL" +nav-parent_id: ops +nav-pos: 7 +--- + + +Flink comes with an integrated interactive Scala Shell. +It can be used in a local setup as well as in a cluster setup. + +To use the shell with an integrated Flink cluster just execute: + +{% highlight bash %} +bin/start-scala-shell.sh local +{% endhighlight %} + +in the root directory of your binary Flink directory. To run the Shell on a +cluster, please see the Setup section below. + +## Usage + +The shell supports DataSet, DataStream, Table API and SQL. +Four different Environments are automatically prebound after startup. +Use "benv" and "senv" to access the Batch and Streaming ExecutionEnvironment respectively. +Use "btenv" and "stenv" to access BatchTableEnvironment and StreamTableEnvironment respectively. + +### DataSet API + +The following example will execute the wordcount program in the Scala shell: + +{% highlight scala %} +Scala-Flink> val text = benv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,") +Scala-Flink> val counts = text + .flatMap { _.toLowerCase.split("\\W+") } + .map { (_, 1) }.groupBy(0).sum(1) +Scala-Flink> counts.print() +{% endhighlight %} + +The print() command will automatically send the specified tasks to the JobManager for execution and will show the result of the computation in the terminal. + +It is possible to write results to a file. However, in this case you need to call `execute`, to run your program: + +{% highlight scala %} +Scala-Flink> benv.execute("MyProgram") +{% endhighlight %} + +### DataStream API + +Similar to the batch program above, we can execute a streaming program through the DataStream API: + +{% highlight scala %} +Scala-Flink> val textStreaming = senv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,") +Scala-Flink> val countsStreaming = textStreaming + .flatMap { _.toLowerCase.split("\\W+") } + .map { (_, 1) }.keyBy(0).sum(1) +Scala-Flink> countsStreaming.print() +Scala-Flink> senv.execute("Streaming Wordcount") +{% endhighlight %} + +Note, that in the Streaming case, the print operation does not trigger execution directly. + +The Flink Shell comes with command history and auto-completion. + +### Table API + +The example below is a wordcount program using Table API: +
    +
    +{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = stenv.fromDataStream( + senv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> val split = new $Split +Scala-Flink> textSource.join(split('text) as 'word). + groupBy('word).select('word, 'word.count as 'count). + toRetractStream[(String, Long)].print +Scala-Flink> senv.execute("Table Wordcount") +{% endhighlight %} +
    +
    +{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = btenv.fromDataSet( + benv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> val split = new $Split +Scala-Flink> textSource.join(split('text) as 'word). + groupBy('word).select('word, 'word.count as 'count). + toDataSet[(String, Long)].print +{% endhighlight %} +
    +
    + +Note, that using $ as a prefix for the class name of TableFunction is a workaround of the issue that scala incorrectly generated inner class name. + +### SQL + +The following example is a wordcount program written in SQL: +
    +
    +{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = stenv.fromDataStream( + senv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> stenv.registerTable("text_source", textSource) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> stenv.registerFunction("split", new $Split) +Scala-Flink> val result = stenv.sqlQuery("""SELECT T.word, count(T.word) AS `count` + FROM text_source + JOIN LATERAL table(split(text)) AS T(word) + ON TRUE + GROUP BY T.word""") +Scala-Flink> result.toRetractStream[(String, Long)].print +Scala-Flink> senv.execute("SQL Wordcount") +{% endhighlight %} +
    +
    +{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = btenv.fromDataSet( + benv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> btenv.registerTable("text_source", textSource) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> btenv.registerFunction("split", new $Split) +Scala-Flink> val result = btenv.sqlQuery("""SELECT T.word, count(T.word) AS `count` + FROM text_source + JOIN LATERAL table(split(text)) AS T(word) + ON TRUE + GROUP BY T.word""") +Scala-Flink> result.toDataSet[(String, Long)].print +{% endhighlight %} +
    +
    + +## Adding external dependencies + +It is possible to add external classpaths to the Scala-shell. These will be sent to the Jobmanager automatically alongside your shell program, when calling execute. + +Use the parameter `-a ` or `--addclasspath ` to load additional classes. + +{% highlight bash %} +bin/start-scala-shell.sh [local | remote | yarn] --addclasspath +{% endhighlight %} + + +## Setup + +To get an overview of what options the Scala Shell provides, please use + +{% highlight bash %} +bin/start-scala-shell.sh --help +{% endhighlight %} + +### Local + +To use the shell with an integrated Flink cluster just execute: + +{% highlight bash %} +bin/start-scala-shell.sh local +{% endhighlight %} + + +### Remote + +To use it with a running cluster start the scala shell with the keyword `remote` +and supply the host and port of the JobManager with: + +{% highlight bash %} +bin/start-scala-shell.sh remote +{% endhighlight %} + +### Yarn Scala Shell cluster + +The shell can deploy a Flink cluster to YARN, which is used exclusively by the +shell. The number of YARN containers can be controlled by the parameter `-n `. +The shell deploys a new Flink cluster on YARN and connects the +cluster. You can also specify options for YARN cluster such as memory for +JobManager, name of YARN application, etc. + +For example, to start a Yarn cluster for the Scala Shell with two TaskManagers +use the following: + +{% highlight bash %} + bin/start-scala-shell.sh yarn -n 2 +{% endhighlight %} + +For all other options, see the full reference at the bottom. + + +### Yarn Session + +If you have previously deployed a Flink cluster using the Flink Yarn Session, +the Scala shell can connect with it using the following command: + +{% highlight bash %} + bin/start-scala-shell.sh yarn +{% endhighlight %} + + +## Full Reference + +{% highlight bash %} +Flink Scala Shell +Usage: start-scala-shell.sh [local|remote|yarn] [options] ... + +Command: local [options] +Starts Flink scala shell with a local Flink cluster + -a | --addclasspath + Specifies additional jars to be used in Flink +Command: remote [options] +Starts Flink scala shell connecting to a remote cluster + + Remote host name as string + + Remote port as integer + + -a | --addclasspath + Specifies additional jars to be used in Flink +Command: yarn [options] +Starts Flink scala shell connecting to a yarn cluster + -n arg | --container arg + Number of YARN container to allocate (= Number of TaskManagers) + -jm arg | --jobManagerMemory arg + Memory for JobManager container with optional unit (default: MB) + -nm | --name + Set a custom name for the application on YARN + -qu | --queue + Specifies YARN queue + -s | --slots + Number of slots per TaskManager + -tm | --taskManagerMemory + Memory per TaskManager container with optional unit (default: MB) + -a | --addclasspath + Specifies additional jars to be used in Flink + --configDir + The configuration directory. + -h | --help + Prints this usage text +{% endhighlight %} + +{% top %} diff --git a/docs/ops/security-kerberos.md b/docs/ops/security-kerberos.md index 5589057a3276ba..0a831b6baec357 100644 --- a/docs/ops/security-kerberos.md +++ b/docs/ops/security-kerberos.md @@ -53,7 +53,7 @@ Flink provides first-class support for Kerberos authentication only. The follow Note that it is possible to enable the use of Kerberos independently for each service or connector. For example, the user may enable Hadoop security without necessitating the use of Kerberos for ZooKeeper, or vice versa. The shared element is the configuration of -Kerbreros credentials, which is then explicitly used by each component. +Kerberos credentials, which is then explicitly used by each component. The internal architecture is based on security modules (implementing `org.apache.flink.runtime.security.modules.SecurityModule`) which are installed at startup. The following sections describes each security module. diff --git a/docs/ops/security-ssl.md b/docs/ops/security-ssl.md index 961af892f5826d..77099ebf61f458 100644 --- a/docs/ops/security-ssl.md +++ b/docs/ops/security-ssl.md @@ -22,124 +22,271 @@ specific language governing permissions and limitations under the License. --> -This page provides instructions on how to enable SSL for the network communication between different flink components. - -## SSL Configuration - -SSL can be enabled for all network communication between flink components. SSL keystores and truststore has to be deployed on each flink node and configured (conf/flink-conf.yaml) using keys in the security.ssl.* namespace (Please see the [configuration page](config.html) for details). SSL can be selectively enabled/disabled for different transports using the following flags. These flags are only applicable when security.ssl.enabled is set to true. - -* **taskmanager.data.ssl.enabled**: SSL flag for data communication between task managers -* **blob.service.ssl.enabled**: SSL flag for blob service client/server communication -* **akka.ssl.enabled**: SSL flag for the akka based control connection between the flink client, jobmanager and taskmanager -* **jobmanager.web.ssl.enabled**: Flag to enable https access to the jobmanager's web frontend - -## Deploying Keystores and Truststores - -You need to have a Java Keystore generated and copied to each node in the flink cluster. The common name or subject alternative names in the certificate should match the node's hostname and IP address. Keystores and truststores can be generated using the keytool utility (https://docs.oracle.com/javase/8/docs/technotes/tools/unix/keytool.html). All flink components should have read access to the keystore and truststore files. - -### Example: Creating self signed CA and keystores for a 2 node cluster - -Execute the following keytool commands to create a truststore with a self signed CA - -~~~ -keytool -genkeypair -alias ca -keystore ca.keystore -dname "CN=Sample CA" -storepass password -keypass password -keyalg RSA -ext bc=ca:true -keytool -keystore ca.keystore -storepass password -alias ca -exportcert > ca.cer -keytool -importcert -keystore ca.truststore -alias ca -storepass password -noprompt -file ca.cer -~~~ - -Now create keystores for each node with certificates signed by the above CA. Let node1.company.org and node2.company.org be the hostnames with IPs 192.168.1.1 and 192.168.1.2 respectively - -#### Node 1 -~~~ -keytool -genkeypair -alias node1 -keystore node1.keystore -dname "CN=node1.company.org" -ext SAN=dns:node1.company.org,ip:192.168.1.1 -storepass password -keypass password -keyalg RSA -keytool -certreq -keystore node1.keystore -storepass password -alias node1 -file node1.csr -keytool -gencert -keystore ca.keystore -storepass password -alias ca -ext SAN=dns:node1.company.org,ip:192.168.1.1 -infile node1.csr -outfile node1.cer -keytool -importcert -keystore node1.keystore -storepass password -file ca.cer -alias ca -noprompt -keytool -importcert -keystore node1.keystore -storepass password -file node1.cer -alias node1 -noprompt -~~~ - -#### Node 2 -~~~ -keytool -genkeypair -alias node2 -keystore node2.keystore -dname "CN=node2.company.org" -ext SAN=dns:node2.company.org,ip:192.168.1.2 -storepass password -keypass password -keyalg RSA -keytool -certreq -keystore node2.keystore -storepass password -alias node2 -file node2.csr -keytool -gencert -keystore ca.keystore -storepass password -alias ca -ext SAN=dns:node2.company.org,ip:192.168.1.2 -infile node2.csr -outfile node2.cer -keytool -importcert -keystore node2.keystore -storepass password -file ca.cer -alias ca -noprompt -keytool -importcert -keystore node2.keystore -storepass password -file node2.cer -alias node2 -noprompt -~~~ - -## Standalone Deployment -Configure each node in the standalone cluster to pick up the keystore and truststore files present in the local file system. - -### Example: 2 node cluster - -* Generate 2 keystores, one for each node, and copy them to the filesystem on the respective node. Also copy the pulic key of the CA (which was used to sign the certificates in the keystore) as a Java truststore on both the nodes -* Configure conf/flink-conf.yaml to pick up these files - -#### Node 1 -~~~ -security.ssl.enabled: true -security.ssl.keystore: /usr/local/node1.keystore -security.ssl.keystore-password: abc123 -security.ssl.key-password: abc123 -security.ssl.truststore: /usr/local/ca.truststore -security.ssl.truststore-password: abc123 -~~~ - -#### Node 2 -~~~ -security.ssl.enabled: true -security.ssl.keystore: /usr/local/node2.keystore -security.ssl.keystore-password: abc123 -security.ssl.key-password: abc123 -security.ssl.truststore: /usr/local/ca.truststore -security.ssl.truststore-password: abc123 -~~~ - -* Restart the flink components to enable SSL for all of flink's internal communication -* Verify by accessing the jobmanager's UI using https url. The task manager's path in the UI should show akka.ssl.tcp:// as the protocol -* The blob server and task manager's data communication can be verified from the log files - -## YARN Deployment -The keystores and truststore can be deployed in a YARN setup in multiple ways depending on the cluster setup. Following are 2 ways to achieve this - -### 1. Deploy keystores before starting the YARN session -The keystores and truststore should be generated and deployed on all nodes in the YARN setup where flink components can potentially be executed. The same flink config file from the flink YARN client is used for all the flink components running in the YARN cluster. Therefore we need to ensure the keystore is deployed and accessible using the same filepath in all the YARN nodes. - -#### Example config -~~~ -security.ssl.enabled: true -security.ssl.keystore: /usr/local/node.keystore -security.ssl.keystore-password: abc123 -security.ssl.key-password: abc123 -security.ssl.truststore: /usr/local/ca.truststore -security.ssl.truststore-password: abc123 -~~~ - -Now you can start the YARN session from the CLI like you would normally do. - -### 2. Use YARN cli to deploy the keystores and truststore -We can use the YARN client's ship files option (-yt) to distribute the keystores and truststore. Since the same keystore will be deployed at all nodes, we need to ensure a single certificate in the keystore can be served for all nodes. This can be done by either using the Subject Alternative Name(SAN) extension in the certificate and setting it to cover all nodes (hostname and ip addresses) in the cluster or by using wildcard subdomain names (if the cluster is setup accordingly). - -#### Example -* Supply the following parameters to the keytool command when generating the keystore: -ext SAN=dns:node1.company.org,ip:192.168.1.1,dns:node2.company.org,ip:192.168.1.2 -* Copy the keystore and the CA's truststore into a local directory (at the cli's working directory), say deploy-keys/ -* Update the configuration to pick up the files from a relative path - -~~~ -security.ssl.enabled: true -security.ssl.keystore: deploy-keys/node.keystore -security.ssl.keystore-password: password -security.ssl.key-password: password -security.ssl.truststore: deploy-keys/ca.truststore -security.ssl.truststore-password: password -~~~ - -* Start the YARN session using the -yt parameter - -~~~ -flink run -m yarn-cluster -yt deploy-keys/ TestJob.jar -~~~ - -When deployed using YARN, flink's web dashboard is accessible through YARN proxy's Tracking URL. To ensure that the YARN proxy is able to access flink's https url you need to configure YARN proxy to accept flink's SSL certificates. Add the custom CA certificate into Java's default truststore on the YARN Proxy node. +* ToC +{:toc} + +This page provides instructions on how to enable TLS/SSL authentication and encryption for network communication with and between Flink processes. + +## Internal and External Connectivity + +When securing network connections between machines processes through authentication and encryption, Apache Flink differentiates between *internal* and *external* connectivity. +*Internal Connectivity* refers to all connections made between Flink processes. These connections run Flink custom protocols. Users never connect directly to internal connectivity endpoints. +*External / REST Connectivity* endpoints refers to all connections made from the outside to Flink processes. This includes the web UI and REST commands to +start and control running Flink jobs/applications, including the communication of the Flink CLI with the JobManager / Dispatcher. + +For more flexibility, security for internal and external connectivity can be enabled and configured separately. + +
    + Internal and External Connectivity +
    + +### Internal Connectivity + +Internal connectivity includes: + + - Control messages: RPC between JobManager / TaskManager / Dispatcher / ResourceManager + - The data plane: The connections between TaskManagers to exchange data during shuffles, broadcasts, redistribution, etc. + - The Blob Service (distribution of libraries and other artifacts). + +All internal connections are SSL authenticated and encrypted. The connections use **mutual authentication**, meaning both server +and client side of each connection need to present the certificate to each other. The certificate acts effectively as a shared +secret. + +A common setup is to generate a dedicated certificate (may be self-signed) for a Flink deployment. The certificate for internal communication +is not needed by any other party to interact with Flink, and can be simply added to the container images, or attached to the YARN deployment. + +*Note: Because internal connections are mutually authenticated with shared certificates, Flink can skip hostname verification. This makes container-based setups easier.* + +### External / REST Connectivity + +All external connectivity is exposed via an HTTP/REST endpoint, used for example by the web UI and the CLI: + + - Communication with the *Dispatcher* to submit jobs (session clusters) + - Communication with the *JobManager* to inspect and modify a running job/application + +The REST endpoints can be configured to require SSL connections. The server will, however, accept connections from any client by default, meaning the REST endpoint does not authenticate the client. + +Simple mutual authentication may be enabled by configuration if authentication of connections to the REST endpoint is required, but we recommend to deploy a "side car proxy": +Bind the REST endpoint to the loopback interface (or the pod-local interface in Kubernetes) and start a REST proxy that authenticates and forwards the requests to Flink. +Examples for proxies that Flink users have deployed are [Envoy Proxy](https://www.envoyproxy.io/) or +[NGINX with MOD_AUTH](http://nginx.org/en/docs/http/ngx_http_auth_request_module.html). + +The rationale behind delegating authentication to a proxy is that such proxies offer a wide variety of authentication options and thus better integration into existing infrastructures. + + +### Queryable State + +Connections to the queryable state endpoints is currently not authenticated or encrypted. + + +## Configuring SSL + +SSL can be enabled separately for *internal* and *external* connectivity: + + - **security.ssl.internal.enabled**: Enable SSL for all *internal* connections. + - **security.ssl.rest.enabled**: Enable SSL for *REST / external* connections. + +*Note: For backwards compatibility, the **security.ssl.enabled** option still exists and enables SSL for both internal and REST endpoints.* + +For internal connectivity, you can optionally disable security for different connection types separately. +When `security.ssl.internal.enabled` is set to `true`, you can set the following parameters to `false` to disable SSL for that particular connection type: + + - `taskmanager.data.ssl.enabled`: Data communication between TaskManagers + - `blob.service.ssl.enabled`: Transport of BLOBs from JobManager to TaskManager + - `akka.ssl.enabled`: Akka-based RPC connections between JobManager / TaskManager / ResourceManager + +### Keystores and Truststores + +The SSL configuration requires to configure a **keystore** and a **truststore**. The *keystore* contains the public certificate +(public key) and the private key, while the truststore contains the trusted certificates or the trusted authorities. Both stores +need to be set up such that the truststore trusts the keystore's certificate. + +#### Internal Connectivity + +Because internal communication is mutually authenticated, keystore and truststore typically contain the same dedicated certificate. +The certificate can use wild card hostnames or addresses, because the certificate is expected to be a shared secret and host +names are not verified. It is even possible to use the same file (the keystore) also as the truststore. + +{% highlight yaml %} +security.ssl.internal.keystore: /path/to/file.keystore +security.ssl.internal.keystore-password: keystore_password +security.ssl.internal.key-password: key_password +security.ssl.internal.truststore: /path/to/file.truststore +security.ssl.internal.truststore-password: truststore_password +{% endhighlight %} + +#### REST Endpoints (external connectivity) + +For REST endpoints, by default the keystore is used by the server endpoint, and the truststore is used by the REST clients (including the CLI client) +to accept the server's certificate. In the case where the REST keystore has a self-signed certificate, the truststore must trust that certificate directly. +If the REST endpoint uses a certificate that is signed through a proper certification hierarchy, the roots of that hierarchy should +be in the trust store. + +If mutual authentication is enabled, the keystore and the truststore are used by both, the server endpoint and the REST clients as with internal connectivity. + +{% highlight yaml %} +security.ssl.rest.keystore: /path/to/file.keystore +security.ssl.rest.keystore-password: keystore_password +security.ssl.rest.key-password: key_password +security.ssl.rest.truststore: /path/to/file.truststore +security.ssl.rest.truststore-password: truststore_password +security.ssl.rest.authentication-enabled: false +{% endhighlight %} + +### Cipher suites + +IMPORTANT +The [IETF RFC 7525](https://tools.ietf.org/html/rfc7525) recommends to use a specific set of cipher suites for strong security. +Because these cipher suites were not available on many setups out of the box, Flink's default value is set to a slightly +weaker but more compatible cipher suite. +We recommend that SSL setups update to the stronger cipher suites, if possible, by adding the below entry to the Flink configuration: + +{% highlight yaml %} +security.ssl.algorithms: TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +{% endhighlight %} + +If these cipher suites are not supported on your setup, you will see that Flink processes will not be able to connect to each other. + +### Complete List of SSL Options + +{% include generated/security_configuration.html %} + +## Creating and Deploying Keystores and Truststores + +Keys, Certificates, and the Keystores and Truststores can be generated using the [keytool utility](https://docs.oracle.com/javase/8/docs/technotes/tools/unix/keytool.html). +You need to have an appropriate Java Keystore and Truststore accessible from each node in the Flink cluster. + + - For standalone setups, this means copying the files to each node, or adding them to a shared mounted directory. + - For container based setups, add the keystore and truststore files to the container images. + - For Yarn/Mesos setups, the cluster deployment phase can automatically distribute the keystore and truststore files. + +For the externally facing REST endpoint, the common name or subject alternative names in the certificate should match the node's hostname and IP address. + + +## Example SSL Setup Standalone and Kubernetes + +**Internal Connectivity** + +Execute the following keytool commands to create a key pair in a keystore: + +{% highlight bash %} +keytool -genkeypair -alias flink.internal -keystore internal.keystore -dname "CN=flink.internal" -storepass internal_store_password -keypass internal_key_password -keyalg RSA -keysize 4096 +{% endhighlight %} + +The single key/certificate in the keystore is used the same way by the server and client endpoints (mutual authentication). +The key pair acts as the shared secret for internal security, and we can directly use it as keystore and truststore. + +{% highlight yaml %} +security.ssl.internal.enabled: true +security.ssl.internal.keystore: /path/to/flink/conf/internal.keystore +security.ssl.internal.truststore: /path/to/flink/conf/internal.keystore +security.ssl.internal.keystore-password: internal_store_password +security.ssl.internal.truststore-password: internal_store_password +security.ssl.internal.key-password: internal_key_password +{% endhighlight %} + +**REST Endpoint** + +The REST endpoint may receive connections from external processes, including tools that are not part of Flink (for example curl request to the REST API). +Setting up a proper certificate that is signed though a CA hierarchy may make sense for the REST endpoint. + +However, as mentioned above, the REST endpoint does not authenticate clients and thus typically needs to be secured via a proxy anyways. + +**REST Endpoint (simple self signed certificate)** + +This example shows how to create a simple keystore / truststore pair. The truststore does not contain the primary key and can +be shared with other applications. In this example, *myhost.company.org / ip:10.0.2.15* is the node (or service) for the Flink master. + +{% highlight bash %} +keytool -genkeypair -alias flink.rest -keystore rest.keystore -dname "CN=myhost.company.org" -ext "SAN=dns:myhost.company.org,ip:10.0.2.15" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 -storetype PKCS12 + +keytool -exportcert -keystore rest.keystore -alias flink.rest -storepass rest_keystore_password -file flink.cer + +keytool -importcert -keystore rest.truststore -alias flink.rest -storepass rest_truststore_password -file flink.cer -noprompt +{% endhighlight %} + +{% highlight yaml %} +security.ssl.rest.enabled: true +security.ssl.rest.keystore: /path/to/flink/conf/rest.keystore +security.ssl.rest.truststore: /path/to/flink/conf/rest.truststore +security.ssl.rest.keystore-password: rest_keystore_password +security.ssl.rest.truststore-password: rest_truststore_password +security.ssl.rest.key-password: rest_key_password +{% endhighlight %} + +**REST Endpoint (with a self signed CA)** + +Execute the following keytool commands to create a truststore with a self signed CA. + +{% highlight bash %} +keytool -genkeypair -alias ca -keystore ca.keystore -dname "CN=Sample CA" -storepass ca_keystore_password -keypass ca_key_password -keyalg RSA -keysize 4096 -ext "bc=ca:true" -storetype PKCS12 + +keytool -exportcert -keystore ca.keystore -alias ca -storepass ca_keystore_password -file ca.cer + +keytool -importcert -keystore ca.truststore -alias ca -storepass ca_truststore_password -file ca.cer -noprompt +{% endhighlight %} + +Now create a keystore for the REST endpoint with a certificate signed by the above CA. +Let *flink.company.org / ip:10.0.2.15* be the hostname of the Flink master (JobManager). + +{% highlight bash %} +keytool -genkeypair -alias flink.rest -keystore rest.signed.keystore -dname "CN=flink.company.org" -ext "SAN=dns:flink.company.org" -storepass rest_keystore_password -keypass rest_key_password -keyalg RSA -keysize 4096 -storetype PKCS12 + +keytool -certreq -alias flink.rest -keystore rest.signed.keystore -storepass rest_keystore_password -keypass rest_key_password -file rest.csr + +keytool -gencert -alias ca -keystore ca.keystore -storepass ca_keystore_password -keypass ca_key_password -ext "SAN=dns:flink.company.org,ip:10.0.2.15" -infile rest.csr -outfile rest.cer + +keytool -importcert -keystore rest.signed.keystore -storepass rest_keystore_password -file ca.cer -alias ca -noprompt + +keytool -importcert -keystore rest.signed.keystore -storepass rest_keystore_password -keypass rest_key_password -file rest.cer -alias flink.rest -noprompt +{% endhighlight %} + +Now add the following configuration to your `flink-conf.yaml`: + +{% highlight yaml %} +security.ssl.rest.enabled: true +security.ssl.rest.keystore: /path/to/flink/conf/rest.signed.keystore +security.ssl.rest.truststore: /path/to/flink/conf/ca.truststore +security.ssl.rest.keystore-password: rest_keystore_password +security.ssl.rest.key-password: rest_key_password +security.ssl.rest.truststore-password: ca_truststore_password +{% endhighlight %} + +**Tips to query REST Endpoint with curl utility** + +You can convert the keystore into the `PEM` format using `openssl`: + +{% highlight bash %} +openssl pkcs12 -passin pass:rest_keystore_password -in rest.keystore -out rest.pem -nodes +{% endhighlight %} + +Then you can query REST Endpoint with `curl`: + +{% highlight bash %} +curl --cacert rest.pem flink_url +{% endhighlight %} + +If mutual SSL is enabled: + +{% highlight bash %} +curl --cacert rest.pem --cert rest.pem flink_url +{% endhighlight %} + +## Tips for YARN / Mesos Deployment + +For YARN and Mesos, you can use the tools of Yarn and Mesos to help: + + - Configuring security for internal communication is exactly the same as in the example above. + + - To secure the REST endpoint, you need to issue the REST endpoint's certificate such that it is valid for all hosts + that the Flink master may get deployed to. This can be done with a wild card DNS name, or by adding multiple DNS names. + + - The easiest way to deploy keystores and truststore is by YARN client's *ship files* option (`-yt`). + Copy the keystore and truststore files into a local directory (say `deploy-keys/`) and start the YARN session as + follows: `flink run -m yarn-cluster -yt deploy-keys/ flinkapp.jar` + + - When deployed using YARN, Flink's web dashboard is accessible through YARN proxy's Tracking URL. + To ensure that the YARN proxy is able to access Flink's HTTPS URL, you need to configure YARN proxy to accept Flink's SSL certificates. + For that, add the custom CA certificate into Java's default truststore on the YARN Proxy node. {% top %} diff --git a/docs/ops/state/checkpoints.md b/docs/ops/state/checkpoints.md index d2df4e6cf33cf9..c68bb0f1ed554d 100644 --- a/docs/ops/state/checkpoints.md +++ b/docs/ops/state/checkpoints.md @@ -35,68 +35,63 @@ the same semantics as a failure-free execution. See [Checkpointing]({{ site.baseurl }}/dev/stream/state/checkpointing.html) for how to enable and configure checkpoints for your program. -## Externalized Checkpoints +## Retained Checkpoints -Checkpoints are by default not persisted externally and are only used to -resume a job from failures. They are deleted when a program is cancelled. -You can, however, configure periodic checkpoints to be persisted externally -similarly to [savepoints](savepoints.html). These *externalized checkpoints* -write their meta data out to persistent storage and are *not* automatically -cleaned up when the job fails. This way, you will have a checkpoint around -to resume from if your job fails. +Checkpoints are by default not retained and are only used to resume a +job from failures. They are deleted when a program is cancelled. +You can, however, configure periodic checkpoints to be retained. +Depending on the configuration these *retained* checkpoints are *not* +automatically cleaned up when the job fails or is canceled. +This way, you will have a checkpoint around to resume from if your job fails. -```java +{% highlight java %} CheckpointConfig config = env.getCheckpointConfig(); config.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); -``` +{% endhighlight %} -The `ExternalizedCheckpointCleanup` mode configures what happens with externalized checkpoints when you cancel the job: +The `ExternalizedCheckpointCleanup` mode configures what happens with checkpoints when you cancel the job: -- **`ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION`**: Retain the externalized checkpoint when the job is cancelled. Note that you have to manually clean up the checkpoint state after cancellation in this case. +- **`ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION`**: Retain the checkpoint when the job is cancelled. Note that you have to manually clean up the checkpoint state after cancellation in this case. -- **`ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION`**: Delete the externalized checkpoint when the job is cancelled. The checkpoint state will only be available if the job fails. +- **`ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION`**: Delete the checkpoint when the job is cancelled. The checkpoint state will only be available if the job fails. ### Directory Structure -Similarly to [savepoints](savepoints.html), an externalized checkpoint consists -of a meta data file and, depending on the state back-end, some additional data -files. The **target directory** for the externalized checkpoint's meta data is -determined from the configuration key `state.checkpoints.dir` which, currently, -can only be set via the configuration files. +Similarly to [savepoints](savepoints.html), a checkpoint consists +of a meta data file and, depending on the state backend, some additional data +files. The meta data file and data files are stored in the directory that is +configured via `state.checkpoints.dir` in the configuration files, +and also can be specified for per job in the code. -``` -state.checkpoints.dir: hdfs:///checkpoints/ -``` +#### Configure globally via configuration files -This directory will then contain the checkpoint meta data required to restore -the checkpoint. For the `MemoryStateBackend`, this meta data file will be -self-contained and no further files are needed. +{% highlight yaml %} +state.checkpoints.dir: hdfs:///checkpoints/ +{% endhighlight %} -`FsStateBackend` and `RocksDBStateBackend` write separate data files -and only write the paths to these files into the meta data file. These data -files are stored at the path given to the state back-end during construction. +#### Configure for per job when constructing the state backend -```java +{% highlight java %} env.setStateBackend(new RocksDBStateBackend("hdfs:///checkpoints-data/"); -``` +{% endhighlight %} ### Difference to Savepoints -Externalized checkpoints have a few differences from [savepoints](savepoints.html). They +Checkpoints have a few differences from [savepoints](savepoints.html). They - use a state backend specific (low-level) data format, may be incremental. - do not support Flink specific features like rescaling. -### Resuming from an externalized checkpoint +### Resuming from a retained checkpoint -A job may be resumed from an externalized checkpoint just as from a savepoint +A job may be resumed from a checkpoint just as from a savepoint by using the checkpoint's meta data file instead (see the [savepoint restore guide](../cli.html#restore-a-savepoint)). Note that if the meta data file is not self-contained, the jobmanager needs to have access to the data files it refers to (see [Directory Structure](#directory-structure) above). -```sh +{% highlight shell %} $ bin/flink run -s :checkpointMetaDataPath [:runArgs] -``` +{% endhighlight %} {% top %} diff --git a/docs/ops/state/large_state_tuning.md b/docs/ops/state/large_state_tuning.md index 28ef99ba3eb54b..81d360660754cd 100644 --- a/docs/ops/state/large_state_tuning.md +++ b/docs/ops/state/large_state_tuning.md @@ -132,7 +132,7 @@ Incremental checkpoints can dramatically reduce the checkpointing time in compar recovery time. The core idea is that incremental checkpoints only record all changes to the previous completed checkpoint, instead of producing a full, self-contained backup of the state backend. Like this, incremental checkpoints build upon previous checkpoints. Flink leverages RocksDB's internal backup mechanism in a way that is self-consolidating over time. As a result, the incremental checkpoint history in Flink -does not grow indefinitely, and old checkpoints are eventually subsumed and pruned automatically. ` +does not grow indefinitely, and old checkpoints are eventually subsumed and pruned automatically. While we strongly encourage the use of incremental checkpoints for large state, please note that this is a new feature and currently not enabled by default. To enable this feature, users can instantiate a `RocksDBStateBackend` with the corresponding boolean flag in the constructor set to `true`, e.g.: @@ -142,6 +142,17 @@ by default. To enable this feature, users can instantiate a `RocksDBStateBackend new RocksDBStateBackend(filebackend, true); {% endhighlight %} +**RocksDB Timers** + +For RocksDB, a user can chose whether timers are stored on the heap (default) or inside RocksDB. Heap-based timers can have a better performance for smaller numbers of +timers, while storing timers inside RocksDB offers higher scalability as the number of timers in RocksDB can exceed the available main memory (spilling to disk). + +When using RockDB as state backend, the type of timer storage can be selected through Flink's configuration via option key `state.backend.rocksdb.timer-service.factory`. +Possible choices are `heap` (to store timers on the heap, default) and `rocksdb` (to store timers in RocksDB). + +Note *The combination RocksDB state backend / with incremental checkpoint / with heap-based timers currently does NOT support asynchronous snapshots for the timers state. +Other state like keyed state is still snapshotted asynchronously. Please note that this is not a regression from previous versions and will be resolved with `FLINK-10026`.* + **Passing Options to RocksDB** {% highlight java %} @@ -172,16 +183,15 @@ public class MyOptions implements OptionsFactory { **Predefined Options** Flink provides some predefined collections of option for RocksDB for different settings, which can be set for example via -`RocksDBStateBacked.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM)`. +`RocksDBStateBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM)`. We expect to accumulate more such profiles over time. Feel free to contribute such predefined option profiles when you found a set of options that work well and seem representative for certain workloads. -**Important:** RocksDB is a native library, whose allocated memory not from the JVM, but directly from the process' -native memory. Any memory you assign to RocksDB will have to be accounted for, typically by decreasing the JVM heap size +Note RocksDB is a native library that allocates memory directly from the process, +and not from the JVM. Any memory you assign to RocksDB will have to be accounted for, typically by decreasing the JVM heap size of the TaskManagers by the same amount. Not doing that may result in YARN/Mesos/etc terminating the JVM processes for -allocating more memory than configures. - +allocating more memory than configured. ## Capacity Planning @@ -231,7 +241,101 @@ Compression can be activated through the `ExecutionConfig`: executionConfig.setUseSnapshotCompression(true); {% endhighlight %} -**Notice:** The compression option has no impact on incremental snapshots, because they are using RocksDB's internal +Note The compression option has no impact on incremental snapshots, because they are using RocksDB's internal format which is always using snappy compression out of the box. +## Task-Local Recovery + +### Motivation + +In Flink's checkpointing, each task produces a snapshot of its state that is then written to a distributed store. Each task acknowledges +a successful write of the state to the job manager by sending a handle that describes the location of the state in the distributed store. +The job manager, in turn, collects the handles from all tasks and bundles them into a checkpoint object. + +In case of recovery, the job manager opens the latest checkpoint object and sends the handles back to the corresponding tasks, which can +then restore their state from the distributed storage. Using a distributed storage to store state has two important advantages. First, the storage +is fault tolerant and second, all state in the distributed store is accessible to all nodes and can be easily redistributed (e.g. for rescaling). + +However, using a remote distributed store has also one big disadvantage: all tasks must read their state from a remote location, over the network. +In many scenarios, recovery could reschedule failed tasks to the same task manager as in the previous run (of course there are exceptions like machine +failures), but we still have to read remote state. This can result in *long recovery time for large states*, even if there was only a small failure on +a single machine. + +### Approach + +Task-local state recovery targets exactly this problem of long recovery time and the main idea is the following: for every checkpoint, each task +does not only write task states to the distributed storage, but also keep *a secondary copy of the state snapshot in a storage that is local to +the task* (e.g. on local disk or in memory). Notice that the primary store for snapshots must still be the distributed store, because local storage +does not ensure durability under node failures and also does not provide access for other nodes to redistribute state, this functionality still +requires the primary copy. + +However, for each task that can be rescheduled to the previous location for recovery, we can restore state from the secondary, local +copy and avoid the costs of reading the state remotely. Given that *many failures are not node failures and node failures typically only affect one +or very few nodes at a time*, it is very likely that in a recovery most tasks can return to their previous location and find their local state intact. +This is what makes local recovery effective in reducing recovery time. + +Please note that this can come at some additional costs per checkpoint for creating and storing the secondary local state copy, depending on the +chosen state backend and checkpointing strategy. For example, in most cases the implementation will simply duplicate the writes to the distributed +store to a local file. + +Illustration of checkpointing with task-local recovery. + +### Relationship of primary (distributed store) and secondary (task-local) state snapshots + +Task-local state is always considered a secondary copy, the ground truth of the checkpoint state is the primary copy in the distributed store. This +has implications for problems with local state during checkpointing and recovery: + +- For checkpointing, the *primary copy must be successful* and a failure to produce the *secondary, local copy will not fail* the checkpoint. A checkpoint +will fail if the primary copy could not be created, even if the secondary copy was successfully created. + +- Only the primary copy is acknowledged and managed by the job manager, secondary copies are owned by task managers and their life cycles can be +independent from their primary copies. For example, it is possible to retain a history of the 3 latest checkpoints as primary copies and only keep +the task-local state of the latest checkpoint. + +- For recovery, Flink will always *attempt to restore from task-local state first*, if a matching secondary copy is available. If any problem occurs during +the recovery from the secondary copy, Flink will *transparently retry to recover the task from the primary copy*. Recovery only fails, if primary +and the (optional) secondary copy failed. In this case, depending on the configuration Flink could still fall back to an older checkpoint. + +- It is possible that the task-local copy contains only parts of the full task state (e.g. exception while writing one local file). In this case, +Flink will first try to recover local parts locally, non-local state is restored from the primary copy. Primary state must always be complete and is +a *superset of the task-local state*. + +- Task-local state can have a different format than the primary state, they are not required to be byte identical. For example, it could be even possible +that the task-local state is an in-memory consisting of heap objects, and not stored in any files. + +- If a task manager is lost, the local state from all its task is lost. + +### Configuring task-local recovery + +Task-local recovery is *deactivated by default* and can be activated through Flink's configuration with the key `state.backend.local-recovery` as specified +in `CheckpointingOptions.LOCAL_RECOVERY`. The value for this setting can either be *true* to enable or *false* (default) to disable local recovery. + +### Details on task-local recovery for different state backends + +***Limitation**: Currently, task-local recovery only covers keyed state backends. Keyed state is typically by far the largest part of the state. In the near future, we will +also cover operator state and timers.* + +The following state backends can support task-local recovery. + +- FsStateBackend: task-local recovery is supported for keyed state. The implementation will duplicate the state to a local file. This can introduce additional write costs +and occupy local disk space. In the future, we might also offer an implementation that keeps task-local state in memory. + +- RocksDBStateBackend: task-local recovery is supported for keyed state. For *full checkpoints*, state is duplicated to a local file. This can introduce additional write costs +and occupy local disk space. For *incremental snapshots*, the local state is based on RocksDB's native checkpointing mechanism. This mechanism is also used as the first step +to create the primary copy, which means that in this case no additional cost is introduced for creating the secondary copy. We simply keep the native checkpoint directory around +instead of deleting it after uploading to the distributed store. This local copy can share active files with the working directory of RocksDB (via hard links), so for active +files also no additional disk space is consumed for task-local recovery with incremental snapshots. Using hard links also means that the RocksDB directories must be on +the same physical device as all the configure local recovery directories that can be used to store local state, or else establishing hard links can fail (see FLINK-10954). +Currently, this also prevents using local recovery when RocksDB directories are configured to be located on more than one physical device. + +### Allocation-preserving scheduling + +Task-local recovery assumes allocation-preserving task scheduling under failures, which works as follows. Each task remembers its previous +allocation and *requests the exact same slot* to restart in recovery. If this slot is not available, the task will request a *new, fresh slot* from the resource manager. This way, +if a task manager is no longer available, a task that cannot return to its previous location *will not drive other recovering tasks out of their previous slots*. Our reasoning is +that the previous slot can only disappear when a task manager is no longer available, and in this case *some* tasks have to request a new slot anyways. With our scheduling strategy +we give the maximum number of tasks a chance to recover from their local state and avoid the cascading effect of tasks stealing their previous slots from one another. + +Allocation-preserving scheduling does not work with Flink's legacy mode. + {% top %} diff --git a/docs/ops/state/savepoints.md b/docs/ops/state/savepoints.md index 1a714b3a5d4ce2..e47e5ff5c5071b 100644 --- a/docs/ops/state/savepoints.md +++ b/docs/ops/state/savepoints.md @@ -25,17 +25,29 @@ under the License. * toc {:toc} -## Overview +## What is a Savepoint? How is a Savepoint different from a Checkpoint? -Savepoints are externally stored self-contained checkpoints that you can use to stop-and-resume or update your Flink programs. They use Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html) to create a (non-incremental) snapshot of the state of your streaming program and write the checkpoint data and meta data out to an external file system. - -This page covers all steps involved in triggering, restoring, and disposing savepoints. -For more details on how Flink handles state and failures in general, check out the [State in Streaming Programs]({{ site.baseurl }}/dev/stream/state/index.html) page. +A Savepoint is a consistent image of the execution state of a streaming job, created via Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html). You can use Savepoints to stop-and-resume, fork, +or update your Flink jobs. Savepoints consist of two parts: a directory with (typically large) binary files on stable storage (e.g. HDFS, S3, ...) and a (relatively small) meta data file. The files on stable storage represent the net data of the job's execution state +image. The meta data file of a Savepoint contains (primarily) pointers to all files on stable storage that are part of the Savepoint, in form of absolute paths.
    Attention: In order to allow upgrades between programs and Flink versions, it is important to check out the following section about assigning IDs to your operators.
    +Conceptually, Flink's Savepoints are different from Checkpoints in a similar way that backups are different from recovery logs in traditional database systems. The primary purpose of Checkpoints is to provide a recovery mechanism in case of +unexpected job failures. A Checkpoint's lifecycle is managed by Flink, i.e. a Checkpoint is created, owned, and released by Flink - without user interaction. As a method of recovery and being periodically triggered, two main +design goals for the Checkpoint implementation are i) being as lightweight to create and ii) being as fast to restore from as possible. Optimizations towards those goals can exploit certain properties, e.g. that the job code +doesn't change between the execution attempts. Checkpoints are usually dropped after the job was terminated by the user (except if explicitly configured as retained Checkpoints). + +In contrast to all this, Savepoints are created, owned, and deleted by the user. Their use-case is for planned, manual backup and resume. For example, this could be an update of your Flink version, changing your job graph, +changing parallelism, forking a second job like for a red/blue deployment, and so on. Of course, Savepoints must survive job termination. Conceptually, Savepoints can be a bit more expensive to produce and restore and focus +more on portability and support for the previously mentioned changes to the job. + +Those conceptual differences aside, the current implementations of Checkpoints and Savepoints are basically using the same code and produce the same „format". However, there is currently one exception from this, and we might +introduce more differences in the future. The exception are incremental checkpoints with the RocksDB state backend. They are using some RocksDB internal format instead of Flink’s native savepoint format. This makes them the +first instance of a more lightweight checkpointing mechanism, compared to Savepoints. + ## Assigning Operator IDs It is **highly recommended** that you adjust your programs as described in this section in order to be able to upgrade your programs in the future. The main required change is to manually specify operator IDs via the **`uid(String)`** method. These IDs are used to scope the state of each operator. @@ -59,12 +71,12 @@ If you don't specify the IDs manually they will be generated automatically. You You can think of a savepoint as holding a map of `Operator ID -> State` for each stateful operator: -``` +{% highlight plain %} Operator ID | State ------------+------------------------ source-id | State of StatefulSource mapper-id | State of StatefulMapper -``` +{% endhighlight %} In the above example, the print sink is stateless and hence not part of the savepoint state. By default, we try to map each entry of the savepoint back to the new program. @@ -84,7 +96,7 @@ When triggering a savepoint, a new savepoint directory is created where the data For example with a `FsStateBackend` or `RocksDBStateBackend`: -```sh +{% highlight shell %} # Savepoint target directory /savepoints/ @@ -96,7 +108,7 @@ For example with a `FsStateBackend` or `RocksDBStateBackend`: # Savepoint state /savepoints/savepoint-:shortjobid-:savepointid/... -``` +{% endhighlight %}
    Note: @@ -106,35 +118,40 @@ Please follow FLINK-5 Note that if you use the `MemoryStateBackend`, metadata *and* savepoint state will be stored in the `_metadata` file. Since it is self-contained, you may move the file and restore from any location. +
    + Attention: It is discouraged to move or delete the last savepoint of a running job, because this might interfere with failure-recovery. Savepoints have side-effects on exactly-once sinks, therefore + to ensure exactly-once semantics, if there is no checkpoint after the last savepoint, the savepoint will be used for recovery. +
    + #### Trigger a Savepoint -```sh +{% highlight shell %} $ bin/flink savepoint :jobId [:targetDirectory] -``` +{% endhighlight %} This will trigger a savepoint for the job with ID `:jobId`, and returns the path of the created savepoint. You need this path to restore and dispose savepoints. #### Trigger a Savepoint with YARN -```sh +{% highlight shell %} $ bin/flink savepoint :jobId [:targetDirectory] -yid :yarnAppId -``` +{% endhighlight %} This will trigger a savepoint for the job with ID `:jobId` and YARN application ID `:yarnAppId`, and returns the path of the created savepoint. #### Cancel Job with Savepoint -```sh +{% highlight shell %} $ bin/flink cancel -s [:targetDirectory] :jobId -``` +{% endhighlight %} This will atomically trigger a savepoint for the job with ID `:jobid` and cancel the job. Furthermore, you can specify a target file system directory to store the savepoint in. The directory needs to be accessible by the JobManager(s) and TaskManager(s). ### Resuming from Savepoints -```sh +{% highlight shell %} $ bin/flink run -s :savepointPath [:runArgs] -``` +{% endhighlight %} This submits a job and specifies a savepoint to resume from. You may give a path to either the savepoint's directory or the `_metadata` file. @@ -142,15 +159,15 @@ This submits a job and specifies a savepoint to resume from. You may give a path By default the resume operation will try to map all state of the savepoint back to the program you are restoring with. If you dropped an operator, you can allow to skip state that cannot be mapped to the new program via `--allowNonRestoredState` (short: `-n`) option: -```sh +{% highlight shell %} $ bin/flink run -s :savepointPath -n [:runArgs] -``` +{% endhighlight %} ### Disposing Savepoints -```sh +{% highlight shell %} $ bin/flink savepoint -d :savepointPath -``` +{% endhighlight %} This disposes the savepoint stored in `:savepointPath`. @@ -160,10 +177,10 @@ Note that it is possible to also manually delete a savepoint via regular file sy You can configure a default savepoint target directory via the `state.savepoints.dir` key. When triggering savepoints, this directory will be used to store the savepoint. You can overwrite the default by specifying a custom target directory with the trigger commands (see the [`:targetDirectory` argument](#trigger-a-savepoint)). -```sh +{% highlight yaml %} # Default savepoint target directory state.savepoints.dir: hdfs:///flink/savepoints -``` +{% endhighlight %} If you neither configure a default nor specify a custom target directory, triggering the savepoint will fail. @@ -189,9 +206,9 @@ By default, a savepoint restore will try to match all state back to the restored You can allow non restored state by setting the `--allowNonRestoredState` (short: `-n`) with the run command: -```sh +{% highlight shell %} $ bin/flink run -s :savepointPath -n [:runArgs] -``` +{% endhighlight %} ### What happens if I reorder stateful operators in my job? @@ -211,4 +228,10 @@ If the savepoint was triggered with Flink >= 1.2.0 and using no deprecated state If you are resuming from a savepoint triggered with Flink < 1.2.0 or using now deprecated APIs you first have to migrate your job and savepoint to Flink >= 1.2.0 before being able to change the parallelism. See the [upgrading jobs and Flink versions guide]({{ site.baseurl }}/ops/upgrading.html). +### Can I move the Savepoint files on stable storage? + +The quick answer to this question is currently "no" because the meta data file references the files on stable storage as absolute paths for technical reasons. The longer answer is: if you MUST move the files for some reason there are two +potential approaches as workaround. First, simpler but potentially more dangerous, you can use an editor to find the old path in the meta data file and replace them with the new path. Second, you can use the class +SavepointV2Serializer as starting point to programmatically read, manipulate, and rewrite the meta data file with the new paths. + {% top %} diff --git a/docs/ops/state/state_backends.md b/docs/ops/state/state_backends.md index 3410623d289869..c1678036875f75 100644 --- a/docs/ops/state/state_backends.md +++ b/docs/ops/state/state_backends.md @@ -116,10 +116,13 @@ The RocksDBStateBackend is encouraged for: Note that the amount of state that you can keep is only limited by the amount of disk space available. This allows keeping very large state, compared to the FsStateBackend that keeps state in memory. This also means, however, that the maximum throughput that can be achieved will be lower with -this state backend. +this state backend. All reads/writes from/to this backend have to go through de-/serialization to retrieve/store the state objects, which is also more expensive than always working with the +on-heap representation as the heap-based backends are doing. RocksDBStateBackend is currently the only backend that offers incremental checkpoints (see [here](large_state_tuning.html)). +Certain RocksDB native metrics are available but disabled by default, you can find full documentation [here]({{ site.baseurl }}/ops/config.html#rocksdb-native-metrics) + ## Configuring a State Backend The default state backend, if you specify nothing, is the jobmanager. If you wish to establish a different default for all jobs on your cluster, you can do so by defining a new default state backend in **flink-conf.yaml**. The default state backend can be overridden on a per-job basis, as shown below. @@ -143,20 +146,31 @@ env.setStateBackend(new FsStateBackend("hdfs://namenode:40010/flink/checkpoints"
    +If you want to use the `RocksDBStateBackend`, then you have to add the following dependency to your Flink project. + +{% highlight xml %} + + org.apache.flink + flink-statebackend-rocksdb{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + ### Setting Default State Backend A default state backend can be configured in the `flink-conf.yaml`, using the configuration key `state.backend`. Possible values for the config entry are *jobmanager* (MemoryStateBackend), *filesystem* (FsStateBackend), *rocksdb* (RocksDBStateBackend), or the fully qualified class -name of the class that implements the state backend factory [FsStateBackendFactory](https://github.com/apache/flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackendFactory.java), +name of the class that implements the state backend factory [StateBackendFactory](https://github.com/apache/flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java), such as `org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory` for RocksDBStateBackend. -In the case where the default state backend is set to *filesystem*, the entry `state.backend.fs.checkpointdir` defines the directory where the checkpoint data will be stored. +The `state.checkpoints.dir` option defines the directory to which all backends write checkpoint data and meta data files. +You can find more details about the checkpoint directory structure [here](checkpoints.html#directory-structure). A sample section in the configuration file could look as follows: -~~~ +{% highlight yaml %} # The backend that will be used to store operator state checkpoints state.backend: filesystem @@ -164,7 +178,11 @@ state.backend: filesystem # Directory for storing checkpoints -state.backend.fs.checkpointdir: hdfs://namenode:40010/flink/checkpoints -~~~ +state.checkpoints.dir: hdfs://namenode:40010/flink/checkpoints +{% endhighlight %} + +#### RocksDB State Backend Config Options + +{% include generated/rocks_db_configuration.html %} {% top %} diff --git a/docs/ops/upgrading.md b/docs/ops/upgrading.md index f5c8ee02e0974d..22cea4c53f0d0c 100644 --- a/docs/ops/upgrading.md +++ b/docs/ops/upgrading.md @@ -36,21 +36,21 @@ The line of action for upgrading a streaming application or migrating an applica There are two ways of taking a savepoint from a running streaming application. * Taking a savepoint and continue processing. -``` +{% highlight bash %} > ./bin/flink savepoint [pathToSavepoint] -``` +{% endhighlight %} It is recommended to periodically take savepoints in order to be able to restart an application from a previous point in time. * Taking a savepoint and stopping the application as a single action. -``` +{% highlight bash %} > ./bin/flink cancel -s [pathToSavepoint] -``` +{% endhighlight %} This means that the application is canceled immediately after the savepoint completed, i.e., no other checkpoints are taken after the savepoint. Given a savepoint taken from an application, the same or a compatible application (see [Application State Compatibility](#application-state-compatibility) section below) can be started from that savepoint. Starting an application from a savepoint means that the state of its operators is initialized with the operator state persisted in the savepoint. This is done by starting an application using a savepoint. -``` +{% highlight bash %} > ./bin/flink run -d -s [pathToSavepoint] ~/application.jar -``` +{% endhighlight %} The operators of the started application are initialized with the operator state of the original application (i.e., the application the savepoint was taken from) at the time when the savepoint was taken. The started application continues processing from exactly this point on. @@ -66,10 +66,10 @@ In this section, we discuss how applications can be modified to remain state com When an application is restarted from a savepoint, Flink matches the operator state stored in the savepoint to stateful operators of the started application. The matching is done based on operator IDs, which are also stored in the savepoint. Each operator has a default ID that is derived from the operator's position in the application's operator topology. Hence, an unmodified application can always be restarted from one of its own savepoints. However, the default IDs of operators are likely to change if an application is modified. Therefore, modified applications can only be started from a savepoint if the operator IDs have been explicitly specified. Assigning IDs to operators is very simple and done using the `uid(String)` method as follows: -``` +{% highlight scala%} val mappedEvents: DataStream[(Int, Long)] = events - .map(new MyStatefulMapFunc()).uid(“mapper-1”) -``` + .map(new MyStatefulMapFunc()).uid("mapper-1") +{% endhighlight %} **Note:** Since the operator IDs stored in a savepoint and IDs of operators in the application to start must be equal, it is highly recommended to assign unique IDs to all operators of an application that might be upgraded in the future. This advice applies to all operators, i.e., operators with and without explicitly declared operator state, because some operators have internal state that is not visible to the user. Upgrading an application without assigned operator IDs is significantly more difficult and may only be possible via a low-level workaround using the `setUidHash()` method. @@ -107,7 +107,7 @@ When upgrading an application by changing its topology, a few things need to be * **Adding a stateful operator:** The state of the operator will be initialized with the default state unless it takes over the state of another operator. * **Removing a stateful operator:** The state of the removed operator is lost unless another operator takes it over. When starting the upgraded application, you have to explicitly agree to discard the state. * **Changing of input and output types of operators:** When adding a new operator before or behind an operator with internal state, you have to ensure that the input or output type of the stateful operator is not modified to preserve the data type of the internal operator state (see above for details). -* **Changing operator chaining:** Operators can be chained together for improved performance. When restoring from a savepoint taken since 1.3.x it is possible to modify chains while preserving state consistency. It is possible a break the chain such that a stateful operator is moved out of the chain. It is also possible to append or inject a new or existing stateful operator into a chain, or to modify the operator order within a chain. However, when upgrading a savepoint to 1.3.x it is paramount that the topology did not change in regards to chaining. All operators that are part of a chain should be assigned an ID as described in the [Matching Operator State](#Matching Operator State) section above. +* **Changing operator chaining:** Operators can be chained together for improved performance. When restoring from a savepoint taken since 1.3.x it is possible to modify chains while preserving state consistency. It is possible a break the chain such that a stateful operator is moved out of the chain. It is also possible to append or inject a new or existing stateful operator into a chain, or to modify the operator order within a chain. However, when upgrading a savepoint to 1.3.x it is paramount that the topology did not change in regards to chaining. All operators that are part of a chain should be assigned an ID as described in the [Matching Operator State](#matching-operator-state) section above. ## Upgrading the Flink Framework Version @@ -141,7 +141,7 @@ about the steps that we outlined before. ### Preconditions Before starting the migration, please check that the jobs you are trying to migrate are following the -best practises for [savepoints]({{ site.baseurl }}/ops/state/savepoints.html). Also, check out the +best practices for [savepoints]({{ site.baseurl }}/ops/state/savepoints.html). Also, check out the [API Migration Guides]({{ site.baseurl }}/dev/migration.html) to see if there is any API changes related to migrating savepoints to newer versions. @@ -171,9 +171,9 @@ savepoints can be relocated using typical filesystem operations.. First major step in job migration is taking a savepoint of your job running in the older Flink version. You can do this with the command: -```sh +{% highlight shell %} $ bin/flink savepoint :jobId [:targetDirectory] -``` +{% endhighlight %} For more details, please read the [savepoint documentation]({{ site.baseurl }}/ops/state/savepoints.html). @@ -190,9 +190,9 @@ If you are unfamiliar with installing Flink in your cluster, please read the [de As the last step of job migration, you resume from the savepoint taken above on the updated cluster. You can do this with the command: -```sh +{% highlight shell %} $ bin/flink run -s :savepointPath [:runArgs] -``` +{% endhighlight %} Again, for more details, please take a look at the [savepoint documentation]({{ site.baseurl }}/ops/state/savepoints.html). @@ -209,6 +209,10 @@ Savepoints are compatible across Flink versions as indicated by the table below: 1.1.x 1.2.x 1.3.x + 1.4.x + 1.5.x + 1.6.x + 1.7.x Limitations @@ -218,7 +222,11 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O O - The maximum parallelism of a job that was migrated from Flink 1.1.x to 1.2.x is + + + + + The maximum parallelism of a job that was migrated from Flink 1.1.x to 1.2.x+ is currently fixed as the parallelism of the job. This means that the parallelism can not be increased after migration. This limitation might be removed in a future bugfix release. @@ -227,15 +235,70 @@ Savepoints are compatible across Flink versions as indicated by the table below: O O - When migrating from Flink 1.2.x to Flink 1.3.x, changing parallelism at the same - time is not supported. Users have to first take a savepoint after migrating to Flink 1.3.x, and then change - parallelism. + O + O + O + O + When migrating from Flink 1.2.x to Flink 1.3.x+, changing parallelism at the same + time is not supported. Users have to first take a savepoint after migrating to Flink 1.3.x+, and then change + parallelism. Savepoints created for CEP applications cannot be restored in 1.4.x+. 1.3.x O + O + O + O + O + Migrating from Flink 1.3.0 to Flink 1.4.[0,1] will fail if the savepoint contains Scala case classes. Users have to directly migrate to 1.4.2+ instead. + + + 1.4.x + + + + O + O + O + O + + + + 1.5.x + + + + + O + O + O + There is a known issue with resuming broadcast state created with 1.5.x in versions + 1.6.x up to 1.6.2, and 1.7.0: FLINK-11087. Users + upgrading to 1.6.x or 1.7.x series need to directly migrate to minor versions higher than 1.6.2 and 1.7.0, + respectively. + + + 1.6.x + + + + + + O + O + + + + 1.7.x + + + + + + + O diff --git a/docs/page/js/jquery.min.js b/docs/page/js/jquery.min.js new file mode 100644 index 00000000000000..e6a051d0d1d327 --- /dev/null +++ b/docs/page/js/jquery.min.js @@ -0,0 +1,4 @@ +/*! jQuery v1.11.2 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l="1.11.2",m=function(a,b){return new m.fn.init(a,b)},n=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,o=/^-ms-/,p=/-([\da-z])/gi,q=function(a,b){return b.toUpperCase()};m.fn=m.prototype={jquery:l,constructor:m,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=m.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return m.each(this,a,b)},map:function(a){return this.pushStack(m.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},m.extend=m.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||m.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(m.isPlainObject(c)||(b=m.isArray(c)))?(b?(b=!1,f=a&&m.isArray(a)?a:[]):f=a&&m.isPlainObject(a)?a:{},g[d]=m.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},m.extend({expando:"jQuery"+(l+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===m.type(a)},isArray:Array.isArray||function(a){return"array"===m.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return!m.isArray(a)&&a-parseFloat(a)+1>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==m.type(a)||a.nodeType||m.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(k.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&m.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(o,"ms-").replace(p,q)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=r(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(n,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(r(Object(a))?m.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=r(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),m.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||m.guid++,e):void 0},now:function(){return+new Date},support:k}),m.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function r(a){var b=a.length,c=m.type(a);return"function"===c||m.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var s=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);m.find=s,m.expr=s.selectors,m.expr[":"]=m.expr.pseudos,m.unique=s.uniqueSort,m.text=s.getText,m.isXMLDoc=s.isXML,m.contains=s.contains;var t=m.expr.match.needsContext,u=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,v=/^.[^:#\[\.,]*$/;function w(a,b,c){if(m.isFunction(b))return m.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return m.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(v.test(b))return m.filter(b,a,c);b=m.filter(b,a)}return m.grep(a,function(a){return m.inArray(a,b)>=0!==c})}m.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?m.find.matchesSelector(d,a)?[d]:[]:m.find.matches(a,m.grep(b,function(a){return 1===a.nodeType}))},m.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(m(a).filter(function(){for(b=0;e>b;b++)if(m.contains(d[b],this))return!0}));for(b=0;e>b;b++)m.find(a,d[b],c);return c=this.pushStack(e>1?m.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(w(this,a||[],!1))},not:function(a){return this.pushStack(w(this,a||[],!0))},is:function(a){return!!w(this,"string"==typeof a&&t.test(a)?m(a):a||[],!1).length}});var x,y=a.document,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=m.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||x).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof m?b[0]:b,m.merge(this,m.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:y,!0)),u.test(c[1])&&m.isPlainObject(b))for(c in b)m.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=y.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return x.find(a);this.length=1,this[0]=d}return this.context=y,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):m.isFunction(a)?"undefined"!=typeof x.ready?x.ready(a):a(m):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),m.makeArray(a,this))};A.prototype=m.fn,x=m(y);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};m.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!m(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),m.fn.extend({has:function(a){var b,c=m(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(m.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=t.test(a)||"string"!=typeof a?m(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&m.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?m.unique(f):f)},index:function(a){return a?"string"==typeof a?m.inArray(this[0],m(a)):m.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(m.unique(m.merge(this.get(),m(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}m.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return m.dir(a,"parentNode")},parentsUntil:function(a,b,c){return m.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return m.dir(a,"nextSibling")},prevAll:function(a){return m.dir(a,"previousSibling")},nextUntil:function(a,b,c){return m.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return m.dir(a,"previousSibling",c)},siblings:function(a){return m.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return m.sibling(a.firstChild)},contents:function(a){return m.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:m.merge([],a.childNodes)}},function(a,b){m.fn[a]=function(c,d){var e=m.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=m.filter(d,e)),this.length>1&&(C[a]||(e=m.unique(e)),B.test(a)&&(e=e.reverse())),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return m.each(a.match(E)||[],function(a,c){b[c]=!0}),b}m.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):m.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){m.each(b,function(b,c){var d=m.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&m.each(arguments,function(a,c){var d;while((d=m.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?m.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},m.extend({Deferred:function(a){var b=[["resolve","done",m.Callbacks("once memory"),"resolved"],["reject","fail",m.Callbacks("once memory"),"rejected"],["notify","progress",m.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return m.Deferred(function(c){m.each(b,function(b,f){var g=m.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&m.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?m.extend(a,d):d}},e={};return d.pipe=d.then,m.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&m.isFunction(a.promise)?e:0,g=1===f?a:m.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&m.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;m.fn.ready=function(a){return m.ready.promise().done(a),this},m.extend({isReady:!1,readyWait:1,holdReady:function(a){a?m.readyWait++:m.ready(!0)},ready:function(a){if(a===!0?!--m.readyWait:!m.isReady){if(!y.body)return setTimeout(m.ready);m.isReady=!0,a!==!0&&--m.readyWait>0||(H.resolveWith(y,[m]),m.fn.triggerHandler&&(m(y).triggerHandler("ready"),m(y).off("ready")))}}});function I(){y.addEventListener?(y.removeEventListener("DOMContentLoaded",J,!1),a.removeEventListener("load",J,!1)):(y.detachEvent("onreadystatechange",J),a.detachEvent("onload",J))}function J(){(y.addEventListener||"load"===event.type||"complete"===y.readyState)&&(I(),m.ready())}m.ready.promise=function(b){if(!H)if(H=m.Deferred(),"complete"===y.readyState)setTimeout(m.ready);else if(y.addEventListener)y.addEventListener("DOMContentLoaded",J,!1),a.addEventListener("load",J,!1);else{y.attachEvent("onreadystatechange",J),a.attachEvent("onload",J);var c=!1;try{c=null==a.frameElement&&y.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!m.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}I(),m.ready()}}()}return H.promise(b)};var K="undefined",L;for(L in m(k))break;k.ownLast="0"!==L,k.inlineBlockNeedsLayout=!1,m(function(){var a,b,c,d;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",k.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(d))}),function(){var a=y.createElement("div");if(null==k.deleteExpando){k.deleteExpando=!0;try{delete a.test}catch(b){k.deleteExpando=!1}}a=null}(),m.acceptData=function(a){var b=m.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var M=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,N=/([A-Z])/g;function O(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(N,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:M.test(c)?m.parseJSON(c):c}catch(e){}m.data(a,b,c)}else c=void 0}return c}function P(a){var b;for(b in a)if(("data"!==b||!m.isEmptyObject(a[b]))&&"toJSON"!==b)return!1; +return!0}function Q(a,b,d,e){if(m.acceptData(a)){var f,g,h=m.expando,i=a.nodeType,j=i?m.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||m.guid++:h),j[k]||(j[k]=i?{}:{toJSON:m.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=m.extend(j[k],b):j[k].data=m.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[m.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[m.camelCase(b)])):f=g,f}}function R(a,b,c){if(m.acceptData(a)){var d,e,f=a.nodeType,g=f?m.cache:a,h=f?a[m.expando]:m.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){m.isArray(b)?b=b.concat(m.map(b,m.camelCase)):b in d?b=[b]:(b=m.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!P(d):!m.isEmptyObject(d))return}(c||(delete g[h].data,P(g[h])))&&(f?m.cleanData([a],!0):k.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}m.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?m.cache[a[m.expando]]:a[m.expando],!!a&&!P(a)},data:function(a,b,c){return Q(a,b,c)},removeData:function(a,b){return R(a,b)},_data:function(a,b,c){return Q(a,b,c,!0)},_removeData:function(a,b){return R(a,b,!0)}}),m.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=m.data(f),1===f.nodeType&&!m._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=m.camelCase(d.slice(5)),O(f,d,e[d])));m._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){m.data(this,a)}):arguments.length>1?this.each(function(){m.data(this,a,b)}):f?O(f,a,m.data(f,a)):void 0},removeData:function(a){return this.each(function(){m.removeData(this,a)})}}),m.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=m._data(a,b),c&&(!d||m.isArray(c)?d=m._data(a,b,m.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=m.queue(a,b),d=c.length,e=c.shift(),f=m._queueHooks(a,b),g=function(){m.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return m._data(a,c)||m._data(a,c,{empty:m.Callbacks("once memory").add(function(){m._removeData(a,b+"queue"),m._removeData(a,c)})})}}),m.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},W=/^(?:checkbox|radio)$/i;!function(){var a=y.createElement("input"),b=y.createElement("div"),c=y.createDocumentFragment();if(b.innerHTML="
    a",k.leadingWhitespace=3===b.firstChild.nodeType,k.tbody=!b.getElementsByTagName("tbody").length,k.htmlSerialize=!!b.getElementsByTagName("link").length,k.html5Clone="<:nav>"!==y.createElement("nav").cloneNode(!0).outerHTML,a.type="checkbox",a.checked=!0,c.appendChild(a),k.appendChecked=a.checked,b.innerHTML="",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,c.appendChild(b),b.innerHTML="",k.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,k.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){k.noCloneEvent=!1}),b.cloneNode(!0).click()),null==k.deleteExpando){k.deleteExpando=!0;try{delete b.test}catch(d){k.deleteExpando=!1}}}(),function(){var b,c,d=y.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(k[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),k[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var X=/^(?:input|select|textarea)$/i,Y=/^key/,Z=/^(?:mouse|pointer|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=/^([^.]*)(?:\.(.+)|)$/;function ab(){return!0}function bb(){return!1}function cb(){try{return y.activeElement}catch(a){}}m.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=m.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof m===K||a&&m.event.triggered===a.type?void 0:m.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(E)||[""],h=b.length;while(h--)f=_.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=m.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=m.event.special[o]||{},l=m.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&m.expr.match.needsContext.test(e),namespace:p.join(".")},i),(n=g[o])||(n=g[o]=[],n.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?n.splice(n.delegateCount++,0,l):n.push(l),m.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m.hasData(a)&&m._data(a);if(r&&(k=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=_.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=m.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,n=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=n.length;while(f--)g=n[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(n.splice(f,1),g.selector&&n.delegateCount--,l.remove&&l.remove.call(a,g));i&&!n.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||m.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)m.event.remove(a,o+b[j],c,d,!0);m.isEmptyObject(k)&&(delete r.handle,m._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,n,o=[d||y],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||y,3!==d.nodeType&&8!==d.nodeType&&!$.test(p+m.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[m.expando]?b:new m.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:m.makeArray(c,[b]),k=m.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!m.isWindow(d)){for(i=k.delegateType||p,$.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||y)&&o.push(l.defaultView||l.parentWindow||a)}n=0;while((h=o[n++])&&!b.isPropagationStopped())b.type=n>1?i:k.bindType||p,f=(m._data(h,"events")||{})[b.type]&&m._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&m.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&m.acceptData(d)&&g&&d[p]&&!m.isWindow(d)){l=d[g],l&&(d[g]=null),m.event.triggered=p;try{d[p]()}catch(r){}m.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=m.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(m._data(this,"events")||{})[a.type]||[],k=m.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=m.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((m.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?m(c,this).index(i)>=0:m.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),hb=/^\s+/,ib=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,jb=/<([\w:]+)/,kb=/\s*$/g,rb={option:[1,""],legend:[1,"
    ","
    "],area:[1,"",""],param:[1,"",""],thead:[1,"","
    "],tr:[2,"","
    "],col:[2,"","
    "],td:[3,"","
    "],_default:k.htmlSerialize?[0,"",""]:[1,"X
    ","
    "]},sb=db(y),tb=sb.appendChild(y.createElement("div"));rb.optgroup=rb.option,rb.tbody=rb.tfoot=rb.colgroup=rb.caption=rb.thead,rb.th=rb.td;function ub(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==K?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==K?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||m.nodeName(d,b)?f.push(d):m.merge(f,ub(d,b));return void 0===b||b&&m.nodeName(a,b)?m.merge([a],f):f}function vb(a){W.test(a.type)&&(a.defaultChecked=a.checked)}function wb(a,b){return m.nodeName(a,"table")&&m.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function xb(a){return a.type=(null!==m.find.attr(a,"type"))+"/"+a.type,a}function yb(a){var b=pb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function zb(a,b){for(var c,d=0;null!=(c=a[d]);d++)m._data(c,"globalEval",!b||m._data(b[d],"globalEval"))}function Ab(a,b){if(1===b.nodeType&&m.hasData(a)){var c,d,e,f=m._data(a),g=m._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)m.event.add(b,c,h[c][d])}g.data&&(g.data=m.extend({},g.data))}}function Bb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!k.noCloneEvent&&b[m.expando]){e=m._data(b);for(d in e.events)m.removeEvent(b,d,e.handle);b.removeAttribute(m.expando)}"script"===c&&b.text!==a.text?(xb(b).text=a.text,yb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),k.html5Clone&&a.innerHTML&&!m.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&W.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}m.extend({clone:function(a,b,c){var d,e,f,g,h,i=m.contains(a.ownerDocument,a);if(k.html5Clone||m.isXMLDoc(a)||!gb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(tb.innerHTML=a.outerHTML,tb.removeChild(f=tb.firstChild)),!(k.noCloneEvent&&k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||m.isXMLDoc(a)))for(d=ub(f),h=ub(a),g=0;null!=(e=h[g]);++g)d[g]&&Bb(e,d[g]);if(b)if(c)for(h=h||ub(a),d=d||ub(f),g=0;null!=(e=h[g]);g++)Ab(e,d[g]);else Ab(a,f);return d=ub(f,"script"),d.length>0&&zb(d,!i&&ub(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,l,n=a.length,o=db(b),p=[],q=0;n>q;q++)if(f=a[q],f||0===f)if("object"===m.type(f))m.merge(p,f.nodeType?[f]:f);else if(lb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(jb.exec(f)||["",""])[1].toLowerCase(),l=rb[i]||rb._default,h.innerHTML=l[1]+f.replace(ib,"<$1>")+l[2],e=l[0];while(e--)h=h.lastChild;if(!k.leadingWhitespace&&hb.test(f)&&p.push(b.createTextNode(hb.exec(f)[0])),!k.tbody){f="table"!==i||kb.test(f)?""!==l[1]||kb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)m.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}m.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),k.appendChecked||m.grep(ub(p,"input"),vb),q=0;while(f=p[q++])if((!d||-1===m.inArray(f,d))&&(g=m.contains(f.ownerDocument,f),h=ub(o.appendChild(f),"script"),g&&zb(h),c)){e=0;while(f=h[e++])ob.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=m.expando,j=m.cache,l=k.deleteExpando,n=m.event.special;null!=(d=a[h]);h++)if((b||m.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)n[e]?m.event.remove(d,e):m.removeEvent(d,e,g.handle);j[f]&&(delete j[f],l?delete d[i]:typeof d.removeAttribute!==K?d.removeAttribute(i):d[i]=null,c.push(f))}}}),m.fn.extend({text:function(a){return V(this,function(a){return void 0===a?m.text(this):this.empty().append((this[0]&&this[0].ownerDocument||y).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?m.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||m.cleanData(ub(c)),c.parentNode&&(b&&m.contains(c.ownerDocument,c)&&zb(ub(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&m.cleanData(ub(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&m.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return m.clone(this,a,b)})},html:function(a){return V(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(fb,""):void 0;if(!("string"!=typeof a||mb.test(a)||!k.htmlSerialize&&gb.test(a)||!k.leadingWhitespace&&hb.test(a)||rb[(jb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(ib,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(m.cleanData(ub(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,m.cleanData(ub(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,n=this,o=l-1,p=a[0],q=m.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&nb.test(p))return this.each(function(c){var d=n.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(i=m.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=m.map(ub(i,"script"),xb),f=g.length;l>j;j++)d=i,j!==o&&(d=m.clone(d,!0,!0),f&&m.merge(g,ub(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,m.map(g,yb),j=0;f>j;j++)d=g[j],ob.test(d.type||"")&&!m._data(d,"globalEval")&&m.contains(h,d)&&(d.src?m._evalUrl&&m._evalUrl(d.src):m.globalEval((d.text||d.textContent||d.innerHTML||"").replace(qb,"")));i=c=null}return this}}),m.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){m.fn[a]=function(a){for(var c,d=0,e=[],g=m(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),m(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Cb,Db={};function Eb(b,c){var d,e=m(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:m.css(e[0],"display");return e.detach(),f}function Fb(a){var b=y,c=Db[a];return c||(c=Eb(a,b),"none"!==c&&c||(Cb=(Cb||m("