Merge pull request #56 from pnuu/feature-tle-download-db

Add a script to download TLEs and store them to a database

.gitignore

@@ -13,7 +13,6 @@ dist
 build
 eggs
 parts
-bin
 var
 sdist
 develop-eggs
@@ -39,4 +38,4 @@ nosetests.xml
 .pydevproject
 
 # rope
-.ropeproject
+.ropeproject

.pre-commit-config.yaml

@@ -0,0 +1,8 @@
+exclude: '^$'
+fail_fast: false
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+    - id: flake8
+      additional_dependencies: [flake8-docstrings, flake8-debugger, flake8-bugbear]

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
-- '2.7'
-- '3.6'
+- '3.7'
+- '3.8'
 install:
 - pip install dask[array] xarray
 - pip install .

bin/fetch_tles.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+"""Script to download and store satellite TLE data."""
+
+import sys
+import logging
+import logging.config
+
+import yaml
+from pyorbital.tlefile import Downloader, SQLiteTLE
+
+
+def read_config(config_fname):
+    """Read and parse config file."""
+    with open(config_fname, 'r') as fid:
+        config = yaml.load(fid, Loader=yaml.SafeLoader)
+    return config
+
+
+def main():
+    """Run TLE downloader."""
+    config = read_config(sys.argv[1])
+    if 'logging' in config:
+        logging.config.dictConfig(config['logging'])
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    downloader = Downloader(config)
+    db = SQLiteTLE(config['database']['path'], config['platforms'],
+                   config['text_writer'])
+
+    logging.info("Start downloading TLEs")
+    for dl_ in config['downloaders']:
+        fetcher = getattr(downloader, dl_)
+        tles = fetcher()
+        if isinstance(tles, dict):
+            for source in tles:
+                for tle in tles[source]:
+                    db.update_db(tle, source)
+        else:
+            source = 'file'
+            if "spacetrack" in dl_:
+                source = 'spacetrack'
+            for tle in tles:
+                db.update_db(tle, source)
+
+    db.write_tle_txt()
+    db.close()
+    logging.info("TLE downloading finished")
+
+
+if __name__ == "__main__":
+    main()

doc/source/index.rst

@@ -27,6 +27,21 @@ Pyorbital has a module for parsing NORAD TLE-files
     99.043499999999995
 
 If no path is given pyorbital tries to read the earth observation TLE-files from celestrak.com
+
+TLE download and database
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There is also a script, ``fetch_tles.py``, that can be used to collect
+TLE data from several locations.  Then currently supported locaions
+are:
+
+* generic network locations without login
+* Space-Track (login credentials needed)
+* local files
+
+The data are saved in a SQLite3 database, and can be written to a file
+after each run.  To see configuration options, see the example
+configuration in ``examples/tle.yaml``.
 
 Computing satellite position
 ----------------------------

examples/tle.yaml

@@ -0,0 +1,63 @@
+# Settings for the TLE dataset
+database:
+  # Path to the dataset where all the data are saved.
+  path: /tmp/tle.db
+
+text_writer:
+  # Directory to save to. Created if missing.
+  output_dir: "/tmp/%Y-%m"
+  # Pattern of the filenames to write.
+  filename_pattern: "tle_%Y%m%d_%H%M.txt"
+  # Write the platform name before the TLE data. Default: False.
+  write_name: False
+  # Write the text file after every invocation. Default: False
+  write_always: False
+
+platforms:
+  # Satellite NORAD ID numbers and corresponding platform names
+  # Only IDs listed here will be added to database and saved to text files
+  25338: NOAA-15
+  28654: NOAA-18
+  33591: NOAA-19
+  37849: Suomi-NPP
+  43013: NOAA-20
+  29499: Metop-A
+  38771: Metop-B
+  43689: Metop-C
+
+downloaders:
+  fetch_plain_tle:
+    eumetsat:  # This is a name used for the source in logs
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_m02_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_m01_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_m03_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_n15_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_n18_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_n19_tle.txt
+      - http://oiswww.eumetsat.int/metopTLEs/html/data_out/latest_npp_tle.txt
+    celestrak:  # This is a name used for the source in logs
+      - https://www.celestrak.com/NORAD/elements/weather.txt
+  fetch_spacetrack:
+    user: <username>
+    password: <password>
+  read_tle_files:
+    # For "kickstarting" the database, local files can also be added
+    paths:
+      - /path/to/a/file/tle.txt
+      - /path/to/many/files/tle*.txt
+
+logging:
+  version: 1
+  formatters:
+    fmt:
+      format: '[%(asctime)s %(levelname)-8s %(name)s] %(message)s'
+  handlers:
+    console:
+      class: logging.StreamHandler
+      level: DEBUG
+      formatter: fmt
+      # stream: ext://sys.stdout
+  root:
+    level: DEBUG
+    propagate: false
+    handlers: [console]