From bd20a7b1fd5d031fc91893ec1512a4f6936b4eb1 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Sat, 2 Mar 2019 21:55:58 +0100 Subject: [PATCH 1/4] Add prettier --- .gitignore | 4 +++- .prettierrc.json | 9 +++++++ package.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 .prettierrc.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 6f317eb..b206152 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .idea target/ *.iml -scalastyle-output.xml \ No newline at end of file +scalastyle-output.xml +/node_modules/ +package-lock.json \ No newline at end of file diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..8238852 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,9 @@ +{ + "arrowParens": "always", + "bracketSpacing": true, + "singleQuote": true, + "parser": "flow", + "printWidth": 120, + "trailingComma": "none", + "tabWidth": 4 +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..f3362ff --- /dev/null +++ b/package.json @@ -0,0 +1,62 @@ +{ + "name": "fiware-cosmos-orion-flink-connector", + "version": "7.6.0", + "description": "enables easier BigData analysis over context integrated with some of the most popular BigData platforms", + "keywords": [ + "big-data", + "fiware", + "flink", + "fiware-cosmos" + ], + "license": "AGPL-3.0", + "repository": { + "type": "git", + "url": "https://github.com/ging/fiware-cosmos-orion-flink-connector" + }, + "devDependencies": { + "eslint": "^5.6.0", + "eslint-config-tamia": "^6.0.1", + "eslint-plugin-prettier": "^3.0.0", + "husky": "^1.1.0", + "lint-staged": "^7.3.0", + "memfs": "2.10.1", + "prettier": "^1.16.4", + "remark-cli": "^6.0.1", + "remark-preset-lint-recommended": "^3.0.2", + "textlint": "^11.0.1", + "textlint-filter-rule-comments": "^1.2.2", + "textlint-rule-common-misspellings": "^1.0.1", + "textlint-rule-no-dead-link": "^4.4.1", + "textlint-rule-terminology": "^1.1.29", + "textlint-rule-write-good": "^1.6.2" + }, + "engines": { + "node": ">=8.6" + }, + "scripts": { + "precommit": "lint-staged", + "lint:text": "textlint 'README.md' 'doc/*.md' 'doc/**/*.md'", + "lint:md": "remark -f 'README.md' 'doc'", + "prettier:text": "prettier 'README.md' 'doc/*.md' 'doc/**/*.md' --no-config --tab-width 4 --print-width 120 --write --prose-wrap always" + }, + "husky": { + "hooks": { + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "*.md": [ + "prettier --no-config --tab-width 4 --print-width 120 --write --prose-wrap always", + "git add" + ] + }, + "remarkConfig": { + "settings": { + "bullet": "-", + "paddedTable": true + }, + "plugins": [ + "remark-preset-lint-recommended" + ] + } +} From cd8f900cb3bcb6f05c90034f516d4a13baabe8d5 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Sat, 2 Mar 2019 22:58:46 +0100 Subject: [PATCH 2/4] Run prettier over documentation --- README.md | 126 +++++++++++++++--------------- doc/contributing.md | 2 +- doc/deprecated_functionalities.md | 15 ++-- doc/getting_started.md | 11 +-- doc/index.md | 40 +++++----- doc/license.md | 1 - doc/qa.md | 23 +++--- doc/roadmap.md | 32 +++++--- doc/testing.md | 3 +- doc/training.md | 13 ++- doc/usage.md | 55 +++++++------ 11 files changed, 172 insertions(+), 149 deletions(-) diff --git a/README.md b/README.md index 0507449..b45a869 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,19 @@ [![Known Vulnerabilities](https://snyk.io/test/github/ging/fiware-cosmos-orion-flink-connector/badge.svg?targetFile=pom.xml)](https://snyk.io/test/github/ging/fiware-cosmos-orion-flink-connector?targetFile=pom.xml) ![Status](https://nexus.lab.fiware.org/static/badges/statuses/cosmos.svg) -The Cosmos Generic Enabler simplifies Big Data analysis of context data and -integrates with some of the many popular Big Data platforms. +The Cosmos Generic Enabler simplifies Big Data analysis of context data and integrates with some of the many popular Big +Data platforms. -Cosmos is a FIWARE Generic Enabler. Therefore, it can be integrated as part of any -platform “Powered by FIWARE”. FIWARE is a curated framework of open source platform -components which can be assembled together with other third-party platform components -to accelerate the development of Smart Solutions. +Cosmos is a FIWARE Generic Enabler. Therefore, it can be integrated as part of any platform “Powered by FIWARE”. FIWARE +is a curated framework of open source platform components which can be assembled together with other third-party +platform components to accelerate the development of Smart Solutions. -This project is part of [FIWARE](https://www.fiware.org/). For more information -check the FIWARE Catalogue entry for +This project is part of [FIWARE](https://www.fiware.org/). For more information check the FIWARE Catalogue entry for [Context Processing, Analysis and Visualization](https://github.com/Fiware/catalogue/tree/master/processing). -| :books: [Documentation](https://fiware-cosmos-flink.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos) | :dart: [Roadmap](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/ROADMAP.md) -| --- | ---|---| +| :books: [Documentation](https://fiware-cosmos-flink.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos) | :dart: [Roadmap](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/ROADMAP.md) | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | + ## Table of Contents @@ -44,8 +43,8 @@ check the FIWARE Catalogue entry for ## What is Cosmos? -The Cosmos Big Data Analysis GE is a set of tools that help achieving the tasks -of Streaming and Batch processing over context data. These tools are: +The Cosmos Big Data Analysis GE is a set of tools that help achieving the tasks of Streaming and Batch processing over +context data. These tools are: - Orion-Flink Connector (Source and Sink) - Apache Flink Processing Engine @@ -54,22 +53,18 @@ of Streaming and Batch processing over context data. These tools are: ## Why use Cosmos? -As the state of the real world changes, the entities representing your IoT -devices are constantly changing. Big data analysis allows for the study of -datasets coming from your context data which are too large for traditional -data-processing software. You can apply predictive analysis or user behaviour -analytics to extract meaningful conclusions as to the state of your smart -solution and bring value to your solution. +As the state of the real world changes, the entities representing your IoT devices are constantly changing. Big data +analysis allows for the study of datasets coming from your context data which are too large for traditional +data-processing software. You can apply predictive analysis or user behaviour analytics to extract meaningful +conclusions as to the state of your smart solution and bring value to your solution. ## Orion Flink Connector This is a Flink connector for the Fiware Orion Context Broker. It has two parts: -- **`OrionSource`**: Source for receiving NGSIv2 events in the shape of HTTP - messages from subscriptions. +- **`OrionSource`**: Source for receiving NGSIv2 events in the shape of HTTP messages from subscriptions. - **`OrionSink`**: Sink for writing back to the Context Broker. - ### Installation Download the JAR from the latest release. In your project directory run: @@ -98,8 +93,7 @@ Add it to your `pom.xml` file inside the dependencies section. import org.fiware.cosmos.orion.flink.connector.{OrionSource} ``` -- Add source to Flink Environment. Indicate what port you want to listen to - (e.g. 9001). +- Add source to Flink Environment. Indicate what port you want to listen to (e.g. 9001). ```scala val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -152,52 +146,55 @@ val processedDataStream = eventStream. // ... The sink accepts a `DataStream` of objects of the class **`OrionSinkObject`**. This class has 4 attributes: -- **`content`**: Message content in String format. If it is a JSON, you need - to make sure to stringify it before sending it. +- **`content`**: Message content in String format. If it is a JSON, you need to make sure to stringify it before + sending it. - **`url`**: URL to which the message should be sent. -- **`contentType`**: Type of HTTP content of the message. It can be - `ContentType.JSON` or `ContentType.Plain`. -- **`method`**: HTTP method of the message. It can be `HTTPMethod.POST`, - `HTTPMethod.PUT` or `HTTPMethod.PATCH`. +- **`contentType`**: Type of HTTP content of the message. It can be `ContentType.JSON` or `ContentType.Plain`. +- **`method`**: HTTP method of the message. It can be `HTTPMethod.POST`, `HTTPMethod.PUT` or `HTTPMethod.PATCH`. #### Production > **Warning** :warning: > -> When packaging your code in a JAR, it is common to exclude dependencies like -> Flink and Scala since they are typically provided by the execution -> environment. Nevertheless, it is necessary to include this connector in your -> packaged code, since it is not part of the Flink distribution. - +> When packaging your code in a JAR, it is common to exclude dependencies like Flink and Scala since they are typically +> provided by the execution environment. Nevertheless, it is necessary to include this connector in your packaged code, +> since it is not part of the Flink distribution. ## Training courses + ### Academy Courses -Some lessons on Big Data Fundamentals are offered in the [FIWARE Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos/) . + +Some lessons on Big Data Fundamentals are offered in the +[FIWARE Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos/) . ### Code Examples -Several examples are provided to facilitate getting started with the connector. They are hosted in a separate repository: [fiware-cosmos-orion-flink-connector-examples](https://github.com/ging/fiware-cosmos-orion-flink-connector-examples). + +Several examples are provided to facilitate getting started with the connector. They are hosted in a separate +repository: +[fiware-cosmos-orion-flink-connector-examples](https://github.com/ging/fiware-cosmos-orion-flink-connector-examples). ### Other Presentations -- [FIWARE Real-time Processing of Historic Context Information using Apache Flink](https://www.slideshare.net/mobile/FI-WARE/fiware-global-summit-fiware-orion-flink-connector) (Málaga 2018) + +- [FIWARE Real-time Processing of Historic Context Information using Apache Flink](https://www.slideshare.net/mobile/FI-WARE/fiware-global-summit-fiware-orion-flink-connector) + (Málaga 2018) ## Quality Assurance -This project is part of [FIWARE](https://fiware.org/) and has been rated as -follows: +This project is part of [FIWARE](https://fiware.org/) and has been rated as follows: - **Version Tested:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Version&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.version&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Version&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.version&colorB=blue) - **Documentation:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Completeness&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docCompleteness&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Usability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docSoundness&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Completeness&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docCompleteness&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Usability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docSoundness&colorB=blue) - **Responsiveness:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Respond&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToCharge&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Fix&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToFix&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Respond&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToCharge&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Fix&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToFix&colorB=blue) - **FIWARE Testing:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Tests%20Passed&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.failureRate&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Scalability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.scalability&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Performance&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.performance&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Stability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.stability&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Tests%20Passed&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.failureRate&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Scalability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.scalability&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Performance&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.performance&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Stability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.stability&colorB=blue) --- @@ -207,38 +204,37 @@ follows: ## Roadmap -The list of features that are planned for the subsequent release are available in the [ROADMAP](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/ROADMAP.md) file. +The list of features that are planned for the subsequent release are available in the +[ROADMAP](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/ROADMAP.md) file. ## Contributing -Contribution guidelines are detailed in the [CONTRIBUTIONS](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/CONTRIBUTIONS.md) file. +Contribution guidelines are detailed in the +[CONTRIBUTIONS](https://github.com/ging/fiware-cosmos-orion-flink-connector/blob/master/CONTRIBUTIONS.md) file. ## Testing In order to test the code run: + ``` mvn clean test -Dtest=*Test cobertura:cobertura coveralls:report -Padd-dependencies-for-IDEA ``` - ## License -Cosmos is licensed under [Affero General Public License (GPL) -version 3](./LICENSE). +Cosmos is licensed under [Affero General Public License (GPL) version 3](./LICENSE). ### Are there any legal issues with AGPL 3.0? Is it safe for me to use? -There is absolutely no problem in using a product licensed under AGPL 3.0. Issues with GPL -(or AGPL) licenses are mostly related with the fact that different people assign different -interpretations on the meaning of the term “derivate work” used in these licenses. Due to this, -some people believe that there is a risk in just _using_ software under GPL or AGPL licenses -(even without _modifying_ it). +There is absolutely no problem in using a product licensed under AGPL 3.0. Issues with GPL (or AGPL) licenses are mostly +related with the fact that different people assign different interpretations on the meaning of the term “derivate work” +used in these licenses. Due to this, some people believe that there is a risk in just _using_ software under GPL or AGPL +licenses (even without _modifying_ it). -For the avoidance of doubt, the owners of this software licensed under an AGPL-3.0 license -wish to make a clarifying public statement as follows: +For the avoidance of doubt, the owners of this software licensed under an AGPL-3.0 license wish to make a clarifying +public statement as follows: -> Please note that software derived as a result of modifying the source code of this -> software in order to fix a bug or incorporate enhancements is considered a derivative -> work of the product. Software that merely uses or aggregates (i.e. links to) an otherwise -> unmodified version of existing software is not considered a derivative work, and therefore -> it does not need to be released as under the same license, or even released as open source. +> Please note that software derived as a result of modifying the source code of this software in order to fix a bug or +> incorporate enhancements is considered a derivative work of the product. Software that merely uses or aggregates (i.e. +> links to) an otherwise unmodified version of existing software is not considered a derivative work, and therefore it +> does not need to be released as under the same license, or even released as open source. diff --git a/doc/contributing.md b/doc/contributing.md index 2fd9f95..a099036 100644 --- a/doc/contributing.md +++ b/doc/contributing.md @@ -1 +1 @@ -TBD \ No newline at end of file +TBD diff --git a/doc/deprecated_functionalities.md b/doc/deprecated_functionalities.md index c39cea8..071f533 100644 --- a/doc/deprecated_functionalities.md +++ b/doc/deprecated_functionalities.md @@ -1,10 +1,9 @@ The following functionalities are no longer maintained: -- TIDOOP -- Apache Hive -- MapReduce -- Cosmos Proxy -- Cosmos GUI -- OAuth2 Hive Authentication Provider -- Auth server - +- TIDOOP +- Apache Hive +- MapReduce +- Cosmos Proxy +- Cosmos GUI +- OAuth2 Hive Authentication Provider +- Auth server diff --git a/doc/getting_started.md b/doc/getting_started.md index 1bdca6c..5a97341 100644 --- a/doc/getting_started.md +++ b/doc/getting_started.md @@ -1,22 +1,23 @@ -This is a Flink connector for the Fiware Orion Context Broker. -It has two parts: +This is a Flink connector for the Fiware Orion Context Broker. It has two parts: - * **`OrionSource`**: Source for receiving NGSIv2 events in the shape of HTTP messages from subscriptions. +- **`OrionSource`**: Source for receiving NGSIv2 events in the shape of HTTP messages from subscriptions. - * **`OrionSink`**: Sink for writing back to the Context Broker. +- **`OrionSink`**: Sink for writing back to the Context Broker. In order to install the connector, first you must download the JAR from the latest release. In your project directory run: + ```bash mvn install:install-file -Dfile=$(PATH_DOWNLOAD)/orion.flink.connector-1.0.jar -DgroupId=org.fiware.cosmos -DartifactId=orion.flink.connector -Dversion=1.0 -Dpackaging=jar ``` Add it to your `pom.xml` file inside the dependencies section. + ```xml org.fiware.cosmos orion.flink.connector 1.0 -``` \ No newline at end of file +``` diff --git a/doc/index.md b/doc/index.md index e3844bf..3723eeb 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,6 +1,6 @@ # fiware-cosmos-orion-flink-connector -[![](https://nexus.lab.fiware.org/static/badges/chapters/processing.svg )](https://www.fiware.org/developers/catalogue/) +[![](https://nexus.lab.fiware.org/static/badges/chapters/processing.svg)](https://www.fiware.org/developers/catalogue/) [![License](https://img.shields.io/github/license/ging/fiware-cosmos-orion-flink-connector.svg)](#) [![](https://img.shields.io/badge/tag-fiware--cosmos-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-cosmos)
@@ -10,32 +10,32 @@ [![Known Vulnerabilities](https://snyk.io/test/github/ging/fiware-cosmos-orion-flink-connector-examples/badge.svg?targetFile=pom.xml)](https://snyk.io/test/github/ging/fiware-cosmos-orion-flink-connector-examples?targetFile=pom.xml) [![Status](https://nexus.lab.fiware.org/static/badges/statuses/cosmos.svg)](#) +The Cosmos Generic Enabler enables an easier BigData analysis over context integrated with some of the most popular +BigData platforms. -The Cosmos Generic Enabler enables an easier BigData analysis over context -integrated with some of the most popular BigData platforms. +Cosmos is a FIWARE Generic Enabler. Therefore, it can be integrated as part of any platform “Powered by FIWARE”. FIWARE +is a curated framework of open source platform components which can be assembled together with other third-party +platform components to accelerate the development of Smart Solutions. -Cosmos is a FIWARE Generic Enabler. Therefore, it can be integrated as part of any platform “Powered by FIWARE”. FIWARE is a curated framework of open source platform components which can be assembled together with other third-party platform components to accelerate the development of Smart Solutions. +You can find more info at the [FIWARE developers](https://developers.fiware.org/) website and the +[FIWARE](https://fiware.org/) website. -You can find more info at the [FIWARE developers](https://developers.fiware.org/) website and the [FIWARE](https://fiware.org/) website. - -The complete list of FIWARE GEs and Incubated FIWARE GEs can be found at the [FIWARE Catalogue](https://catalogue.fiware.org/) +The complete list of FIWARE GEs and Incubated FIWARE GEs can be found at the +[FIWARE Catalogue](https://catalogue.fiware.org/) ## What is Cosmos? -The Cosmos BigData Analysis GE is a set of tools that help achieving the tasks -of Streaming and Batch processing over context data. These tools are: +The Cosmos BigData Analysis GE is a set of tools that help achieving the tasks of Streaming and Batch processing over +context data. These tools are: -* Orion-Flink Connector (Source and Sink) -* Apache Flink Processing Engine -* Apache Spark Processing Engine (work in progress) -* Streaming processing examples using Orion Context Broker +- Orion-Flink Connector (Source and Sink) +- Apache Flink Processing Engine +- Apache Spark Processing Engine (work in progress) +- Streaming processing examples using Orion Context Broker ## Why use Cosmos? -As the state of the real world changes, the entities representing your -IoT devices are constantly changing. Big data analysis allows for the -study of datasets coming from your context data which are too large -for traditional data-processing software. You can apply predictive -analysis or user behaviour analytics to extract meaningful conclusions -as to the state of your smart solution and bring value to your -solution. +As the state of the real world changes, the entities representing your IoT devices are constantly changing. Big data +analysis allows for the study of datasets coming from your context data which are too large for traditional +data-processing software. You can apply predictive analysis or user behaviour analytics to extract meaningful +conclusions as to the state of your smart solution and bring value to your solution. diff --git a/doc/license.md b/doc/license.md index f5ff05d..6781432 100644 --- a/doc/license.md +++ b/doc/license.md @@ -1,4 +1,3 @@ ## License Cosmos is licensed under Affero General Public License (GPL) version 3. - diff --git a/doc/qa.md b/doc/qa.md index 193920f..8cd56eb 100644 --- a/doc/qa.md +++ b/doc/qa.md @@ -1,20 +1,17 @@ -This project is part of [FIWARE](https://fiware.org/) and has been rated as -follows: +This project is part of [FIWARE](https://fiware.org/) and has been rated as follows: - **Version Tested:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Version&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.version&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Version&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.version&colorB=blue) - **Documentation:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Completeness&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docCompleteness&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Usability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docSoundness&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Completeness&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docCompleteness&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Usability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.docSoundness&colorB=blue) - **Responsiveness:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Respond&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToCharge&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Fix&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToFix&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Respond&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToCharge&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Time%20to%20Fix&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.timeToFix&colorB=blue) - **FIWARE Testing:** - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Tests%20Passed&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.failureRate&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Scalability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.scalability&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Performance&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.performance&colorB=blue) - ![ ](https://img.shields.io/badge/dynamic/json.svg?label=Stability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.stability&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Tests%20Passed&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.failureRate&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Scalability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.scalability&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Performance&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.performance&colorB=blue) + ![](https://img.shields.io/badge/dynamic/json.svg?label=Stability&url=https://fiware.github.io/catalogue/json/cosmos.json&query=$.stability&colorB=blue) --- - - \ No newline at end of file diff --git a/doc/roadmap.md b/doc/roadmap.md index 60ea86f..1ed1c30 100644 --- a/doc/roadmap.md +++ b/doc/roadmap.md @@ -1,21 +1,35 @@ -This product is a FIWARE Generic Enabler. If you would like to learn about the overall Roadmap of FIWARE, please check section "Roadmap" on the FIWARE Catalogue. +This product is a FIWARE Generic Enabler. If you would like to learn about the overall Roadmap of FIWARE, please check +section "Roadmap" on the FIWARE Catalogue. ### Introduction -This section elaborates on proposed new features or tasks which are expected to be added to the product in the foreseeable future. There should be no assumption of a commitment to deliver these features on specific dates or in the order given. The development team will be doing their best to follow the proposed dates and priorities, but please bear in mind that plans to work on a given feature or task may be revised. All information is provided as a general guideline only, and this section may be revised to provide newer information at any time. + +This section elaborates on proposed new features or tasks which are expected to be added to the product in the +foreseeable future. There should be no assumption of a commitment to deliver these features on specific dates or in the +order given. The development team will be doing their best to follow the proposed dates and priorities, but please bear +in mind that plans to work on a given feature or task may be revised. All information is provided as a general guideline +only, and this section may be revised to provide newer information at any time. ### Short term -The following list of features are planned to be addressed in the short term, and incorporated in the next release of the product: -* There are no new features planned for the next release. +The following list of features are planned to be addressed in the short term, and incorporated in the next release of +the product: + +- There are no new features planned for the next release. ### Medium term -The following list of features are planned to be addressed in the medium term, typically within the subsequent release(s) generated in the next 9 months after next planned release: - * **Support for Apache Spark**: Right now, the only streaming engine supported by Cosmos is Flink. It is intended to provide an HTTP Orion Connector for Spark as well. - * **Docker images**: Provide Docker images to facilitate the deployment of a complete big data processing scenario. +The following list of features are planned to be addressed in the medium term, typically within the subsequent +release(s) generated in the next 9 months after next planned release: + +- **Support for Apache Spark**: Right now, the only streaming engine supported by Cosmos is Flink. It is intended to + provide an HTTP Orion Connector for Spark as well. +- **Docker images**: Provide Docker images to facilitate the deployment of a complete big data processing scenario. ### Long term -The following list of features are proposals regarding the longer-term evolution of the product even though development of these features has not yet been scheduled for a release in the near future. Please feel free to contact us if you wish to get involved in the implementation or influence the roadmap: +The following list of features are proposals regarding the longer-term evolution of the product even though development +of these features has not yet been scheduled for a release in the near future. Please feel free to contact us if you +wish to get involved in the implementation or influence the roadmap: -* **Data Usage control**: Data usage control consists on keeping track of all the operations performed on data and check if they comply with a set of predefined policies set by the data owner, canceling the streaming job otherwise. +- **Data Usage control**: Data usage control consists on keeping track of all the operations performed on data and + check if they comply with a set of predefined policies set by the data owner, canceling the streaming job otherwise. diff --git a/doc/testing.md b/doc/testing.md index 97cf613..b7e91e7 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -1,4 +1,5 @@ In order to test the code run: + ``` mvn clean test -Dtest=*Test cobertura:cobertura coveralls:report -Padd-dependencies-for-IDEA -``` \ No newline at end of file +``` diff --git a/doc/training.md b/doc/training.md index 50e786f..5f75523 100644 --- a/doc/training.md +++ b/doc/training.md @@ -1,8 +1,15 @@ ### Academy Courses -Some lessons on Big Data Fundamentals are offered in the [FIWARE Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos/) . + +Some lessons on Big Data Fundamentals are offered in the +[FIWARE Academy](https://fiware-academy.readthedocs.io/en/latest/processing/cosmos/) . ### Code Examples -Several examples are provided to facilitate getting started with the connector. They are hosted in a separate repository: [fiware-cosmos-orion-flink-connector-examples](https://github.com/ging/fiware-cosmos-orion-flink-connector-examples). + +Several examples are provided to facilitate getting started with the connector. They are hosted in a separate +repository: +[fiware-cosmos-orion-flink-connector-examples](https://github.com/ging/fiware-cosmos-orion-flink-connector-examples). ### Other Presentations -- [FIWARE Real-time Processing of Historic Context Information using Apache Flink](https://www.slideshare.net/mobile/FI-WARE/fiware-global-summit-fiware-orion-flink-connector) (Málaga 2018) + +- [FIWARE Real-time Processing of Historic Context Information using Apache Flink](https://www.slideshare.net/mobile/FI-WARE/fiware-global-summit-fiware-orion-flink-connector) + (Málaga 2018) diff --git a/doc/usage.md b/doc/usage.md index 0f90739..3f8b491 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1,15 +1,19 @@ ### OrionSource -* Import dependency. +- Import dependency. + ```scala import org.fiware.cosmos.orion.flink.connector.{OrionSource} ``` -* Add source to Flink Environment. Indicate what port you want to listen to (e.g. 9001). + +- Add source to Flink Environment. Indicate what port you want to listen to (e.g. 9001). + ```scala val env = StreamExecutionEnvironment.getExecutionEnvironment val eventStream = env.addSource(new OrionSource(9001)) ``` -* Parse the received data. + +- Parse the received data. ```scala val processedDataStream = eventStream. @@ -19,33 +23,37 @@ val processedDataStream = eventStream. The received data is a DataStream of objects of the class **`NgsiEvent`**. This class has the following attributes: -* **`creationTime`**: Timestamp of arrival. - -* **`service`**: Fiware service extracted from the HTTP headers. +- **`creationTime`**: Timestamp of arrival. -* **`servicePath`**: Fiware service path extracted from the HTTP headers. +- **`service`**: Fiware service extracted from the HTTP headers. -* **`entities`**: Sequence of entites included in the message. Each entity has the following attributes: +- **`servicePath`**: Fiware service path extracted from the HTTP headers. - * **`id`**: Identifier of the entity. +- **`entities`**: Sequence of entites included in the message. Each entity has the following attributes: - * **`type`**: Node type. + - **`id`**: Identifier of the entity. - * **`attrs`**: Map of attributes in which the key is the attribute name and the value is an object with the following properties: + - **`type`**: Node type. - * **`type`**: Type of value (Float, Int,...). + - **`attrs`**: Map of attributes in which the key is the attribute name and the value is an object with the + following properties: - * **`value`**: Value of the attribute. + - **`type`**: Type of value (Float, Int,...). - * **`metadata`**: Additional metadata. + - **`value`**: Value of the attribute. + - **`metadata`**: Additional metadata. ### OrionSink -* Import dependency. + +- Import dependency. + ```scala import org.fiware.cosmos.orion.flink.connector.{OrionSink,OrionSinkObject,ContentType,HTTPMethod} ``` -* Add sink to source. + +- Add sink to source. + ```scala val processedDataStream = eventStream. // ... processing @@ -62,16 +70,17 @@ OrionSink.addSink( processedDataStream ) The sink accepts a `DataStream` of objects of the class **`OrionSinkObject`**. This class has 4 attributes: -- **`content`**: Message content in String format. If it is a JSON, you need to make sure to stringify it before sending it. +- **`content`**: Message content in String format. If it is a JSON, you need to make sure to stringify it before + sending it. -- **`url`**: URL to which the message should be sent. +- **`url`**: URL to which the message should be sent. -- **`contentType`**: Type of HTTP content of the message. It can be `ContentType.JSON` or `ContentType.Plain`. - -- **`method`**: HTTP method of the message. It can be `HTTPMethod.POST`, `HTTPMethod.PUT` or `HTTPMethod.PATCH`. +- **`contentType`**: Type of HTTP content of the message. It can be `ContentType.JSON` or `ContentType.Plain`. +- **`method`**: HTTP method of the message. It can be `HTTPMethod.POST`, `HTTPMethod.PUT` or `HTTPMethod.PATCH`. ### Production -When packaging your code in a JAR, it is common to exclude dependencies like Flink and Scala since they are typically provided by the execution environment. Nevertheless, it is necessary to include this connector in your packaged code, since it is not part of the Flink distribution. - +When packaging your code in a JAR, it is common to exclude dependencies like Flink and Scala since they are typically +provided by the execution environment. Nevertheless, it is necessary to include this connector in your packaged code, +since it is not part of the Flink distribution. From 09461b8558d6882a7f6f8ef0fd3aa0110fb5a88b Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Sat, 2 Mar 2019 23:12:22 +0100 Subject: [PATCH 3/4] Tidy travis --- .travis.yml | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1d4008..afa25af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,33 @@ -language: java -jdk: -- oraclejdk8 -install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Padd-dependencies-for-IDEA +branches: + only: + - master + jobs: include: - stage: test - script: mvn clean test -Dtest=*Test cobertura:cobertura coveralls:report -Padd-dependencies-for-IDEA -DCOVERALLS_TOKEN=$COVERALLS_TOKEN - - stage: lint - script: mvn verify -Padd-dependencies-for-IDEA -after_success: - - mvn clean cobertura:cobertura coveralls:report \ No newline at end of file + name: "Unit Tests: Java 8" + language: java + jdk: + - oraclejdk8 + install: + - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Padd-dependencies-for-IDEA + before_script: + - mvn verify -Padd-dependencies-for-IDEA + script: + - mvn clean test -Dtest=*Test cobertura:cobertura coveralls:report -Padd-dependencies-for-IDEA -DCOVERALLS_TOKEN=$COVERALLS_TOKEN + after_success: + - mvn clean cobertura:cobertura coveralls:report + + - stage: test + name: "Documentation Tests" + language: node_js + + node_js: + - "10" + + install: + - npm install + + script: + - npm run lint:md + - npm run lint:text \ No newline at end of file From fe6108ee76bdae3e397b98158d34178e46d18a9f Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Sat, 2 Mar 2019 23:22:24 +0100 Subject: [PATCH 4/4] Add textlint config. --- .textlintrc | 199 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 .textlintrc diff --git a/.textlintrc b/.textlintrc new file mode 100644 index 0000000..f653de3 --- /dev/null +++ b/.textlintrc @@ -0,0 +1,199 @@ +{ + "rules": { + "terminology": { + "defaultTerms": false, + "terms": [ + // Brands + "Airbnb", + "Android", + "AppleScript", + "AppVeyor", + "AVA", + "BrowserStack", + "Browsersync", + "Codecov", + "CodePen", + "CodeSandbox", + "DefinitelyTyped", + "EditorConfig", + "ESLint", + "FIWARE", + "GitHub", + "GraphQL", + "iOS", + "JavaScript", + "JetBrains", + "jQuery", + "LinkedIn", + "Lodash", + "MacBook", + "Markdown", + "OpenType", + "PayPal", + "PhpStorm", + "RubyMine", + "Sass", + "SemVer", + "TypeScript", + "UglifyJS", + "WebStorm", + "WordPress", + "YouTube", + ["JSDocs?", "JSDoc"], +// ["Node(?:js)?", "Node.js"], + ["React[ .]js", "React"], + ["SauceLabs", "Sauce Labs"], + ["StackOverflow", "Stack Overflow"], + ["styled ?components", "styled-components"], + ["HTTP[ /]2(?:\\.0)?", "HTTP/2"], + ["OS X", "macOS"], + ["Mac ?OS", "macOS"], + ["a npm", "an npm"], + + // ECMAScript + "ECMAScript", + ["ES2015", "ES6"], + ["ES7", "ES2016"], + + // Abbreviations + "3D", + ["3-D", "3D"], + "Ajax", + "API", + ["API['’]?s", "APIs"], + "CSS", + "GIF", + "HTML", + "HTTPS", + "IoT", + "I/O", + ["I-O", "I/O"], + "JPEG", + "MIME", + "OK", + "PaaS", + "PDF", + "PNG", + "SaaS", + "URL", + ["URL['’]?s", "URLs"], + ["an URL", "a URL"], + ["wi[- ]?fi", "Wi-Fi"], + + // Names + "McKenzie", + "McConnell", + + // Words and phrases + "ID", // http://stackoverflow.com/questions/1151338/id-or-id-on-user-interface + ["id['’]?s", "IDs"], + ["backwards compatible", "backward compatible"], + ["build system(s?)", "build tool$1"], + ["CLI tool(s?)", "command-line tool$1"], + ["he or she", "they"], + ["he/she", "they"], + ["\\(s\\)he", "they"], + ["repo\\b", "repository"], + ["smartphone(s?)", "mobile phone$1"], + ["web[- ]?site(s?)", "site$1"], + + // Single word + ["auto[- ]complete", "autocomplete"], + ["auto[- ]format", "autoformat"], + ["auto[- ]fix", "autofix"], + ["auto[- ]fixing", "autofixing"], + ["back[- ]end(\\w*)", "backend$1"], + ["bug[- ]fix(es?)", "bugfix$1"], + ["check[- ]box(es?)", "checkbox$1"], + ["code[- ]base(es?)", "codebase$1"], + ["co[- ]locate(d?)", "colocate$1"], + ["end[- ]point(s?)", "endpoint$1"], + ["e[- ]mail(s?)", "email$1"], + ["file[- ]name(s?)", "filename$1"], + ["front[- ]end(\\w*)", "frontend$1"], + ["hack[- ]a[- ]thon(s?)", "hackathon$1"], + ["host[- ]name(s?)", "hostname$1"], + ["hot[- ]key(s?)", "hotkey$1"], + ["life[- ]cycle", "lifecycle"], + ["life[- ]stream(s?)", "lifestream$1"], + ["lock[- ]file(s?)", "lockfile$1"], + ["mark-up", "markup"], // “mark up” as a verb is OK + ["meta[- ]data", "metadata"], + ["name[- ]space(s?)", "namespace$1"], + ["pre[- ]condition(s?)", "precondition$1"], + ["pre[- ]defined", "predefined"], + ["pre[- ]release(s?)", "prerelease$1"], + ["run[- ]time", "runtime"], + ["screen[- ]shot(s?)", "screenshot$1"], + ["screen[- ]?snap(s?)", "screenshot$1"], + ["sub[- ]class((?:es|ing)?)", "subclass$1"], + ["sub[- ]tree(s?)", "subtree$1"], + ["time[- ]stamp(s?)", "timestamp$1"], + ["touch[- ]screen(s?)", "touchscreen$1"], + ["user[- ]name(s?)", "username$1"], + ["walk[- ]through", "walkthrough"], + ["white[- ]space", "whitespace"], + ["wild[- ]card(s?)", "wildcard$1"], + + // Multiple words + ["change-?log(s?)", "change log$1"], + ["css-?in-?js", "CSS in JS"], + ["code-?review(s?)", "code review$1"], + ["code-?splitting", "code splitting"], + ["end-?user(s?)", "end user$1"], + ["file-?type(s?)", "file type$1"], + ["open-?source(ed?)", "open source$1"], + ["regexp?(s?)", "regular expression$1"], + ["style-?guide(s?)", "style guide$1"], + ["tree-?shaking", "tree shaking"], + ["source-?map(s?)", "source map$1"], + ["style-?sheet(s?)", "style sheet$1"], + ["user-?base", "user base"], + ["web-?page(s?)", "web page$1"], + + // Hyphenated + ["built ?in", "built-in"], + ["client ?side", "client-side"], + ["command ?line", "command-line"], + ["end ?to ?end", "end-to-end"], + ["error ?prone", "error-prone"], + ["higher ?order", "higher-order"], + ["key[/ ]?value", "key-value"], + ["server ?side", "server-side"], + ["two ?steps?", "two-step"], + ["2 ?steps?", "two-step"], + + // Starts from a lower case letter in the middle of a sentence + ["(\\w+[^.?!]\\)? )base64", "$1base64"], + ["(\\w+[^.?!]\\)? )stylelint", "$1stylelint"], + ["(\\w+[^.?!]\\)? )webpack", "$1webpack"], + ["(\\w+[^.?!]\\)? )npm", "$1npm"], + + // Typos + ["environemnt(s?)", "environment$1"], + ["pacakge(s?)", "package$1"], + ["tilda", "tilde"], + ["falsey", "falsy"] + ] + }, + "common-misspellings": true, + "write-good": { + "adverb": false, + "passive": false, + "tooWordy": false, + "weasel": false, + "so": false, + "thereIs": false + }, + "no-dead-link": { + "ignoreRedirects": true, + "ignore": [ + "mailto:*", + "https://oauth.net" + ] + } + }, + "filters": { + "comments": true + } +} \ No newline at end of file