"species" dumper

Author: sirloon

.gitignore

@@ -1,67 +1,14 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-#Ipython Notebook
-.ipynb_checkpoints
-
-# ansible stuff
-.gitmodules
-src/ansible_deploy/
-src/ansible_deploy
+*.pyc
+*.swp
+src/config.py
+__pycache__
+.DS_Store
+logs
+pyenv
+docs/_build
+docs/doc_html
+src/www/static/docs
+src/config_prod.py
+src/run/*.pickle
+src/run/done/*.pickle
+src/bin/ssh_host_key*

src/dataload/__init__.py

@@ -1,3 +1,4 @@
 __sources__ = [
         "dataload.sources.taxonomy",
+        "dataload.sources.species",
 ]

src/dataload/sources/species/__init__.py

@@ -0,0 +1 @@
+from .dumper import SpeciesDumper

src/dataload/sources/species/dumper.py

@@ -0,0 +1,31 @@
+import os
+
+import biothings, config
+biothings.config_for_app(config)
+
+from config import DATA_ARCHIVE_ROOT
+from biothings.dataload.dumper import FTPDumper
+
+
+class SpeciesDumper(FTPDumper):
+
+    SRC_NAME = "species"
+    SRC_ROOT_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, SRC_NAME)
+    FTP_HOST = 'ftp.uniprot.org'
+    CWD_DIR = '/pub/databases/uniprot/current_release/knowledgebase/complete/docs'
+    SUFFIX_ATTR = "timestamp"
+
+    SCHEDULE = "0 9 * * *"
+
+    def create_todump_list(self, force=False):
+        file_to_dump = "speclist.txt"
+        new_localfile = os.path.join(self.new_data_folder,file_to_dump)
+        try:
+            current_localfile = os.path.join(self.current_data_folder, file_to_dump)
+        except TypeError:
+            # current data folder doesn't even exist
+            current_localfile = new_localfile
+        if force or not os.path.exists(current_localfile) or self.remote_is_better(file_to_dump, current_localfile):
+            # register new release (will be stored in backend)
+            self.to_dump.append({"remote": file_to_dump, "local":new_localfile})
+