biothings.species/src/hub/dataload/sources/taxonomy/uploader.py at master · biothings/biothings.species · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import os

import biothings.hub.dataload.uploader as uploader

from .parser import parse_refseq_names, parse_refseq_nodes


class TaxonomyNodesUploader(uploader.BaseSourceUploader):

    main_source = "taxonomy"
    name = "nodes"

    def load_data(self, data_folder):
        nodes_file = os.path.join(data_folder, "nodes.dmp")
        self.logger.info("Load data from file '%s'" % nodes_file)
        return parse_refseq_nodes(open(nodes_file))

    @classmethod
    def get_mapping(klass):
        return {
            "rank": {
                "type": "text"
            },
            "taxid": {
                "copy_to": ["all"],
                "type": "long"
            },
            "parent_taxid": {
                "type": "long"
            },
            "other_names": {
                "type": "text"
            },
        }


class TaxonomyNamesUploader(uploader.BaseSourceUploader):

    main_source = "taxonomy"
    name = "names"
    __metadata__ = {
        "mapper": 'has_gene',
    }

    def load_data(self, data_folder):
        names_file = os.path.join(data_folder, "names.dmp")
        self.logger.info("Load data from file '%s'" % names_file)
        return parse_refseq_names(open(names_file))

    @classmethod
    def get_mapping(klass):
        return {
            "scientific_name": {
                "copy_to": ["all"],
                "type": "text",
                "fields": {
                    "raw": {
                        "type": "keyword",
                        "normalizer": "keyword_lowercase_normalizer"
                    },
                    "raw2": {
                        "type": "keyword"
                    }
                }
            },
            "common_name": {
                "copy_to": ["all"],
                "type": "text",
                "fields": {
                    "raw": {
                        "type": "keyword",
                        "normalizer": "keyword_lowercase_normalizer"
                    },
                    "raw2": {
                        "type": "keyword"
                    }
                }
            },
            "genbank_common_name": {
                "type": "text",
                "fields": {
                    "raw": {
                        "type": "keyword",
                        "normalizer": "keyword_lowercase_normalizer"
                    },
                    "raw2": {
                        "type": "keyword"
                    }
                }
            }
        }