From 6cee5cfcbcdd444ae939f3cd81807f2f78f01d73 Mon Sep 17 00:00:00 2001 From: AleksMat Date: Thu, 24 Jan 2019 19:45:42 +0100 Subject: [PATCH] updates in CONTRIBUTING and README, changed how they are parsed into documentation --- .gitignore | 1 + CONTRIBUTING.md | 37 ++++++++++----------- README.md | 26 +++++++++++---- docs/source/README.md | 67 -------------------------------------- docs/source/conf.py | 45 +++++++++++++++++++++++-- docs/source/contribute.rst | 2 +- docs/source/index.rst | 14 +------- docs/source/install.rst | 18 +--------- 8 files changed, 83 insertions(+), 127 deletions(-) delete mode 100644 docs/source/README.md diff --git a/.gitignore b/.gitignore index f7aaa6fc8..77f394ed9 100644 --- a/.gitignore +++ b/.gitignore @@ -108,6 +108,7 @@ ENV/ # Files that Sphinx generates /docs/source/examples +/docs/source/markdowns /docs/source/eotasks.rst # pytest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81778658b..c9044756d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,28 +5,29 @@ to the library is very much appreciated. Here is how you can contribute: - * [Bug Reports][bug-reports] - * [Feature Requests][feature-requests] - * [Pull Requests][pull-requests] + * [Bug Reports](#bug-reports) + * [Feature Requests](#feature-requests) + * [Pull Requests](#pull-requests) -All contributors agree to follow our [Code of Conduct](code-of-conduct). +All contributors agree to follow our [Code of Conduct][code-of-conduct]. -**eo-learn** is distributed under the [MIT license](LICENSE). When contributing +**eo-learn** is distributed under the [MIT license][license]. When contributing code to the library, you agree to its terms and conditions. If you would like to keep parts of your contribution private, you can contact us to discuss about the best solution. -For any question, feel free to contact us at _eoresearch@sinergise.com_ or through our [Forum][sh-forum]. +For any question, feel free to contact us at [eoresearch@sinergise.com](eoresearch@sinergise.com) or through our [Forum][sh-forum]. +[code-of-conduct]: https://github.com/sentinel-hub/eo-learn/blob/master/CODE_OF_CONDUCT.md +[license]: https://github.com/sentinel-hub/eo-learn/blob/master/LICENSE [sh-forum]: https://forum.sentinel-hub.com/ ## Bug Reports -[bug-reports]: #bug-reports We strive to provide high-quality working code, but bugs happen nevertheless. When reporting a bug, please check [here][open-bug-list] whether -the bug was already reported. If not, open an issue with the **_bug_** label and +the bug was already reported. If not, open an issue with the **bug** label and report the following information: * Issue description @@ -43,11 +44,10 @@ an issue and clarify. [open-bug-list]: https://github.com/sentinel-hub/eo-learn/issues?q=state:open+type:issue+label:"bug" ## Feature Requests -[feature-requests]: #feature-requests Existing feature requests can be found [here][existing-feature-requests]. -A new feature request can be created by opening a new issue with the **_enhancement_** label, +A new feature request can be created by opening a new issue with the **enhancement** label, and describing how the feature would benefit the **eo-learn** community. Providing an example use-case would help assessing the scope of the feature request. @@ -55,7 +55,6 @@ feature request. [existing-feature-requests]: https://github.com/sentinel-hub/eo-learn/issues?q=state:open+type:issue+label:"enhancement" ## Pull Requests -[pull-requests]: #pull-requests The GitHub Pull Request (PR) mechanism is the best option to contribute code to the library. Users can fork the repository, make their contribution to their @@ -67,7 +66,7 @@ Existing PRs can be found [here][existing-prs]. Before creating new PRs, you sho whether someone else has contributed a similar feature, and if so, you can add your input to the existing code review. -The follwoing guidelines should be observed when creating a PR. +The following guidelines should be observed when creating a PR. [fork-and-pull]: https://help.github.com/articles/creating-a-pull-request-from-a-fork [create-pr]: https://help.github.com/articles/creating-a-pull-request/ @@ -80,7 +79,7 @@ The follwoing guidelines should be observed when creating a PR. completion of the code review, the branch will be merged into `develop` and, at the next package release, into `master`. - * Document your PR to help mantainers understand and review your contribution. The PR + * Document your PR to help maintainers understand and review your contribution. The PR should include: * Description of contribution; @@ -88,8 +87,8 @@ The follwoing guidelines should be observed when creating a PR. * Link to issue/feature request. * Your contribution should include unit tests, to test correct behaviour of the new feature - and to lower the mantainance effort. Bug fixes as well as new features should include unit tests. - When submitting the PR, check whether the travis CI testing returns any errors, and if it does, + and to lower the maintenance effort. Bug fixes as well as new features should include unit tests. + When submitting the PR, check whether the Travis CI testing returns any errors, and if it does, please try to fix the issues causing failure. A test `EOPatch` is made available [here][test-eo-patch] with data for each `FeatureType`. Unit tests evaluating the correctness of new tasks should use data available in this `EOPatch`. New fields useful for testing purposes can be added, but should @@ -108,8 +107,7 @@ The follwoing guidelines should be observed when creating a PR. Therefore please run `pylint` from the the main folder, which contains the `pylintrc` file, to make sure your contribution is scored `10.0/10.0`. -#### Contribute an `EOTask` -[create-eo-task]: #create-eo-task +### Contribute an `EOTask` `EOTask`s allow to apply **eo-learn** workflows to different use-cases, adapting to imaging sources and processing chain. If you think a task is general enough to be useful to the community, then we would @@ -141,14 +139,13 @@ When creating a new task, bear in mind the following: and pass task parameters as arguments, either in the constructor, or at run-time. * If in doubt on whether a task is general enough to be of interest to the community, or you are not sure to which sub-package to contribute your task to, send us an email or open a - [feature request][feature-requests]. + [feature request](#feature-requests). Looking forward to include your contributions into **eo-learn**. -[pypi]: https://pypi.org/ +[pypi]: https://pypi.org/project/eo-learn/ [pep-8]: https://www.python.org/dev/peps/pep-0008/ [pylint]: https://www.pylint.org/ [existing-eo-tasks]: https://eo-learn.readthedocs.io/en/latest/eotasks.html -[code-of-conduct][https://github.com/sentinel-hub/eo-learn/blob/master/CODE_OF_CONDUCT.md] [test-eo-patch]: https://github.com/sentinel-hub/eo-learn/tree/master/example_data/TestEOPatch diff --git a/README.md b/README.md index 9ed4b7bb8..fd0116638 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ [![Build Status](https://travis-ci.org/sentinel-hub/eo-learn.svg?branch=master)](https://travis-ci.org/sentinel-hub/eo-learn) [![Docs status](https://readthedocs.org/projects/eo-learn/badge/?version=latest)](https://eo-learn.readthedocs.io) [![License](https://img.shields.io/pypi/l/eo-learn.svg)](https://github.com/sentinel-hub/eo-learn/blob/master/LICENSE) -drawing +[![Overall downloads](http://pepy.tech/badge/eo-learn)](https://pepy.tech/project/eo-learn) +[![Last month downloads](https://pepy.tech/badge/eo-learn/month)](https://pepy.tech/project/eo-learn) + # eo-learn -**`eo-learn` makes extraction of valuable information from satellite imagery easy.** +**eo-learn makes extraction of valuable information from satellite imagery easy.** The availability of open Earth observation (EO) data through the Copernicus and Landsat programs represents an unprecedented resource for many EO applications, ranging from ocean and land use and land cover monitoring, @@ -42,14 +44,13 @@ At the moment there are the following subpackages: ## Installation -The package requires Python version `>=3.5`. It can be installed with: +The package requires Python version **>=3.5** . It can be installed with: ```bash pip install eo-learn ``` -however it is also possible to install each subpackage separately: - +In order to avoid heavy package dependencies it is possible to install each subpackage separately: ```bash pip install eo-learn-core pip install eo-learn-coregistration @@ -60,12 +61,23 @@ pip install eo-learn-mask pip install eo-learn-ml-tools ``` +Before installing `eo-learn` on **Windows** it is recommended to install the following packages from [Unofficial Windows wheels repository](https://www.lfd.uci.edu/~gohlke/pythonlibs/): + +```bash +gdal +rasterio +shapely +fiona +``` + +One of dependecies of `eo-learn-mask` subpackage is `lightgbm` package. If having problems during installation please check [LightGBM installation guide](https://lightgbm.readthedocs.io/en/latest/Installation-Guide.html). + ## Documentation For more information on the package content, visit [readthedocs](https://eo-learn.readthedocs.io/). -# Blog posts +## Blog posts * [Introducing eo-learn](https://medium.com/sentinel-hub/introducing-eo-learn-ab37f2869f5c) * [Land Cover Classification with eo-learn: Part 1 - Mastering Satellite Image Data in an Open-Source Python Environment](https://medium.com/sentinel-hub/land-cover-classification-with-eo-learn-part-1-2471e8098195) @@ -75,4 +87,4 @@ For more information on the package content, visit [readthedocs](https://eo-lear ## License -See [LICENSE](LICENSE). +See [LICENSE](https://github.com/sentinel-hub/eo-learn/blob/master/LICENSE). diff --git a/docs/source/README.md b/docs/source/README.md deleted file mode 100644 index 79365fed9..000000000 --- a/docs/source/README.md +++ /dev/null @@ -1,67 +0,0 @@ -**eo-learn makes extraction of valuable information from satellite imagery easy.** - -The availability of open Earth observation (EO) data through the Copernicus and Landsat programs represents an -unprecedented resource for many EO applications, ranging from ocean and land use and land cover monitoring, -disaster control, emergency services and humanitarian relief. Given the large amount of high spatial resolution -data at high revisit frequency, techniques able to automatically extract complex patterns in such _spatio-temporal_ -data are needed. - -**eo-learn** is a collection of open source Python packages that have been developed to seamlessly access and process -_spatio-temporal_ image sequences acquired by any satellite fleet in a timely and automatic manner. **eo-learn** is -easy to use, it's design modular, and encourages collaboration -- sharing and reusing of specific tasks in a typical -EO-value-extraction workflows, such as cloud masking, image co-registration, feature extraction, classification, etc. Everyone is free -to use any of the available tasks and is encouraged to improve the, develop new ones and share them with the rest of the community. - -**eo-learn** makes extraction of valuable information from satellite imagery as easy as defining a sequence of operations to be performed on satellite imagery. Image below illustrates a processing chain that maps water in satellite imagery by thresholding the Normalised Difference Water Index in user specified region of interest. - -![eo-learn-workflow0illustration](figures/eo-learn-illustration.png) - -**eo-learn** _library acts as a bridge between Earth observation/Remote sensing field and Python ecosystem for data science and machine learning._ The library is written in Python and uses NumPy arrays to store and handle remote sensing data. Its aim is to make entry easier for non-experts to the field of remote sensing on one hand and bring the state-of-the-art tools for computer vision, machine learning, and deep learning existing in Python ecosystem to remote sensing experts. - -## Package Overview - -**eo-learn** is divided into several subpackages according to different functionalities and external package dependencies. Therefore it is not necessary for user to install entire package but only the parts that he needs. - -At the moment there are the following subpackages: - -- **eo-learn-core** - The main subpackage which implements basic building blocks (EOPatch, EOTask and EOWorkflow) and commonly used functionalities. -- **eo-learn-coregistration** - The subpackage that deals with image co-registraion. -- **eo-learn-features** - A collection of utilities for extracting data properties and feature manipulation. -- **eo-learn-geometry** - Geometry subpackage used for geometric transformation and conversion between vector and raster data. -- **eo-learn-io** - Input/output subpackage that deals with obtaining data from Sentinel Hub services or saving and loading data locally. -- **eo-learn-mask** - The subpackage used for masking of data and calculation of cloud masks. -- **eo-learn-ml-tools** - Various tools that can be used before or after the machine learning process. - -## Building blocks - -The design of the **eo-learn** library follows the dataflow programing paradigm and consists of three building blocks: -1. **EOPatch** - * common data-object for _spatio-temporal_ EO and non-EO data, and their derivatives -2. **EOTask** - * a single, well-defined action being performed on existing **EOPatch**(es) - * each **EOTask** takes an **EOPatch** as an input and returns a modified **EOPatch** -3. **EOWorkflow** - * a collection of **EOTask**s that together represent an _EO-value-adding-processing_ chain -4. **EOExecutor** - * Handles execution of a **EOWorkflow** multiple times at once with different input data. - -### EOPatch - -**EOPatch** contains multi-temporal remotely sensed data of a single patch (area) of Earth's surface typically -defined by a bounding box in specific coordinate reference system. The **EOPatch** object can also be used as a placeholder -for all quantities, either derived from the satellite imagery or from some other external source, such as for example -biophysical indices, ground truth reference data, weather data, etc. **EOPatch** is completely sensor-agnostic, meaning that imagery from different sensors (satellites) or sensor types (optical, synthetic-aperture radar, ...) can be added to an **EOPatch**. - -### EOTask - -**EOTask**s are in a sense the heart of the eo-learn library. They define in what way the available satellite imagery can be manipulated in order to extract valuable information out of it. Typical user will most often just be interested in what kind of tasks are already implemented and ready to be used or how to implement an **EOTask**, if it doesn't exist yet. - -### EOWorkflow - -**EOWorkflow** represents the entire EO processing chain or EO-value-extraction pipeline by chaining or connecting sequence of **EOTask**s. The **EOWorkflow** takes care that the **EOTask**s are executed in the correct order and with correct parameters. **EOWorkflow** is executed on a single **EOPatch** at a time, but the same **EOWorkflow** can be executed on multiple **EOTask**s. - -Under the hood the **EOWorkflow** builds a directed acyclic graph (in case user tries to build a cyclic graph the **EOWorkflow** will complain). There's no limitation on the number of nodes (**EOTask**s with inputs) or the graph's topology. The **EOWorkflow** first names the input tasks that persist over executions, determines the ordering of the tasks, executes the task in that order, and finally returns the results of tasks with no outgoing edge. - -### EOExecutor - -**EOExecutor** wraps the entire pipeline process together by executing a **EOWorkflow** as many times as required, each time with different user-defined input parameters. The executions can be parallel or consecutive. During each execution process **EOExecutor** monitors the progress, saves log files and catches any possible pipeline breaking errors. At the end it can produce a report with all information about executions. It provides an easy way to track when something was processed and how to repeat the process with exactly the same parameters. diff --git a/docs/source/conf.py b/docs/source/conf.py index 4325901cb..f66a68e88 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -202,15 +202,56 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/3.6/': None} + +EXAMPLES_FOLDER = './examples' +MARKDOWNS_FOLDER = './markdowns' + # copy examples -shutil.rmtree('./examples', ignore_errors=True) +shutil.rmtree(EXAMPLES_FOLDER, ignore_errors=True) +shutil.rmtree(MARKDOWNS_FOLDER, ignore_errors=True) try: - shutil.copytree('../../examples', './examples') + shutil.copytree('../../examples', EXAMPLES_FOLDER) + os.mkdir(MARKDOWNS_FOLDER) + shutil.copyfile('../../CONTRIBUTING.md', os.path.join(MARKDOWNS_FOLDER, 'CONTRIBUTING.md')) except FileExistsError: pass +def process_readme(): + """ Function which will process README.md file and divide it into INTRO.md and INSTALL.md, which will be used in + documentation + """ + with open('../../README.md', 'r') as file: + readme = file.read() + + readme = readme.replace('# eo-learn', '# Introduction').replace('docs/source/', '') + readme = readme.replace('**`', '**').replace('`**', '**') + + chapters = [[]] + for line in readme.split('\n'): + if line.strip().startswith('## '): + chapters.append([]) + if line.startswith('=3.5`. It can be installed with:: - -$ pip install eo-learn - -however it is also possible to install each subpackage separately:: - -$ pip install eo-learn-core -$ pip install eo-learn-coregistration -$ pip install eo-learn-features -$ pip install eo-learn-geometry -$ pip install eo-learn-io -$ pip install eo-learn-mask -$ pip install eo-learn-ml-tools +.. mdinclude:: markdowns/INSTALL.md