Skip to content

Commit ee674c5

Browse files
committed
"species" dumper
1 parent 82b1507 commit ee674c5

File tree

4 files changed

+47
-67
lines changed

4 files changed

+47
-67
lines changed

.gitignore

Lines changed: 14 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,14 @@
1-
# Byte-compiled / optimized / DLL files
2-
__pycache__/
3-
*.py[cod]
4-
*$py.class
5-
6-
# C extensions
7-
*.so
8-
9-
# Distribution / packaging
10-
.Python
11-
env/
12-
build/
13-
develop-eggs/
14-
dist/
15-
downloads/
16-
eggs/
17-
.eggs/
18-
lib/
19-
lib64/
20-
parts/
21-
sdist/
22-
var/
23-
*.egg-info/
24-
.installed.cfg
25-
*.egg
26-
27-
# PyInstaller
28-
# Usually these files are written by a python script from a template
29-
# before PyInstaller builds the exe, so as to inject date/other infos into it.
30-
*.manifest
31-
*.spec
32-
33-
# Installer logs
34-
pip-log.txt
35-
pip-delete-this-directory.txt
36-
37-
# Unit test / coverage reports
38-
htmlcov/
39-
.tox/
40-
.coverage
41-
.coverage.*
42-
.cache
43-
nosetests.xml
44-
coverage.xml
45-
*,cover
46-
.hypothesis/
47-
48-
# Translations
49-
*.mo
50-
*.pot
51-
52-
# Django stuff:
53-
*.log
54-
55-
# Sphinx documentation
56-
docs/_build/
57-
58-
# PyBuilder
59-
target/
60-
61-
#Ipython Notebook
62-
.ipynb_checkpoints
63-
64-
# ansible stuff
65-
.gitmodules
66-
src/ansible_deploy/
67-
src/ansible_deploy
1+
*.pyc
2+
*.swp
3+
src/config.py
4+
__pycache__
5+
.DS_Store
6+
logs
7+
pyenv
8+
docs/_build
9+
docs/doc_html
10+
src/www/static/docs
11+
src/config_prod.py
12+
src/run/*.pickle
13+
src/run/done/*.pickle
14+
src/bin/ssh_host_key*

src/dataload/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
__sources__ = [
22
"dataload.sources.taxonomy",
3+
"dataload.sources.species",
34
]
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .dumper import SpeciesDumper
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import os
2+
3+
import biothings, config
4+
biothings.config_for_app(config)
5+
6+
from config import DATA_ARCHIVE_ROOT
7+
from biothings.dataload.dumper import FTPDumper
8+
9+
10+
class SpeciesDumper(FTPDumper):
11+
12+
SRC_NAME = "species"
13+
SRC_ROOT_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, SRC_NAME)
14+
FTP_HOST = 'ftp.uniprot.org'
15+
CWD_DIR = '/pub/databases/uniprot/current_release/knowledgebase/complete/docs'
16+
SUFFIX_ATTR = "timestamp"
17+
18+
SCHEDULE = "0 9 * * *"
19+
20+
def create_todump_list(self, force=False):
21+
file_to_dump = "speclist.txt"
22+
new_localfile = os.path.join(self.new_data_folder,file_to_dump)
23+
try:
24+
current_localfile = os.path.join(self.current_data_folder, file_to_dump)
25+
except TypeError:
26+
# current data folder doesn't even exist
27+
current_localfile = new_localfile
28+
if force or not os.path.exists(current_localfile) or self.remote_is_better(file_to_dump, current_localfile):
29+
# register new release (will be stored in backend)
30+
self.to_dump.append({"remote": file_to_dump, "local":new_localfile})
31+

0 commit comments

Comments
 (0)