-
Notifications
You must be signed in to change notification settings - Fork 7
/
article.go
144 lines (135 loc) · 4.55 KB
/
article.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package ios
// Copyright 2023 by Leipzig University Library, http://ub.uni-leipzig.de
// The Finc Authors, http://finc.info
// Martin Czygan, <martin.czygan@uni-leipzig.de>
//
// This file is part of some open source application.
//
// Some open source application is free software: you can redistribute
// it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// Some open source application is distributed in the hope that it will
// be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Foobar. If not, see <http://www.gnu.org/licenses/>.
//
// @license GPL-3.0+ <http://spdx.org/licenses/GPL-3.0+>
import (
"encoding/base64"
"encoding/xml"
"fmt"
"regexp"
"strings"
"github.com/miku/span"
"github.com/miku/span/container"
"github.com/miku/span/formats/finc"
"github.com/miku/span/formats/jats"
"golang.org/x/text/language"
)
const (
// SourceID for internal bookkeeping, refs #24731
SourceID = "219"
// Format for intermediate schema.
Format = "ElectronicArticle"
)
var (
// ArticleTitleBlockPatterns
ArticleTitleBlockPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)(front|back)\s*matter`),
regexp.MustCompile(`(?i)table\s*of\s*content[s]?`),
}
DOIPattern = regexp.MustCompile(`10\.[0-9]+\/\S+`)
// SourceName for finc.mega_collection.
SourceNames = []string{"IOS-Press Zeitschriften", "sid-219-col-ios"}
)
type Article struct {
XMLName xml.Name `xml:"article"`
jats.Article
}
// Identifiers returns identifiers.
func (article *Article) Identifiers() (jats.Identifiers, error) {
doi, err := article.DOI()
if err != nil {
return jats.Identifiers{}, err
}
locator := fmt.Sprintf("https://doi.org/%s", doi)
id := fmt.Sprintf("ai-%s-%s", SourceID, base64.RawURLEncoding.EncodeToString([]byte(doi)))
return jats.Identifiers{DOI: doi, URL: locator, ID: id}, nil
}
// Languages returns a list of language in 3-letter format.
func (article *Article) Languages() []string {
set := container.NewStringSet()
for _, cm := range article.Front.Article.CustomMetaGroup.CustomMeta {
if cm.Name.Value == "lang" {
base, err := language.ParseBase(cm.Value.Value)
if err == nil {
set.Add(base.ISO3())
}
}
}
return set.Values()
}
// ToIntermediateSchema converts an article into an internal schema. There are a
// couple of content-dependent choices here.
func (article *Article) ToIntermediateSchema() (*finc.IntermediateSchema, error) {
output, err := article.Article.ToIntermediateSchema()
if err != nil {
// TODO: in this case, conversion cannot fail
return output, err
}
ids, err := article.Identifiers()
if err == jats.ErrNoDOI {
return output, span.Skip{
Reason: fmt.Sprintf("missing doi, ids: %v", article.Front.Article.ID),
}
}
if err != nil {
return output, err
}
output.DOI = ids.DOI
id := ids.ID
if len(id) > span.KeyLengthLimit {
return output, span.Skip{Reason: fmt.Sprintf("id too long: %s", id)}
}
if len(id) == 0 {
return nil, span.Skip{Reason: fmt.Sprintf("no doi or identifier: %v", article)}
}
output.ID = id
output.RecordID = ids.DOI
output.URL = append(output.URL, ids.URL)
output.Authors = article.Authors()
output.Format = Format
output.Languages = article.Languages()
output.MegaCollections = SourceNames
output.SourceID = SourceID
var normalized []string
for _, issn := range output.ISSN {
if len(issn) == 8 && !strings.Contains(issn, "-") {
normalized = append(normalized, fmt.Sprintf("%s-%s", issn[:4], issn[4:]))
}
}
output.ISSN = normalized
// refs #5686
if output.Date.IsZero() {
return output, span.Skip{Reason: fmt.Sprintf("zero date: %s", output.ID)}
}
// refs #5686
for _, p := range ArticleTitleBlockPatterns {
if p.MatchString(output.ArticleTitle) {
return output, span.Skip{Reason: fmt.Sprintf("title blacklisted: %s", output.ArticleTitle)}
}
}
// refs #5686, approx. article type distribution: https://git.io/vzlCr
// switch article.Type {
// case "book-review", "book-reviews", "Book Review":
// output.ArticleTitle = fmt.Sprintf("Review: %s", article.ReviewedProduct())
// case "misc", "other", "front-matter", "back-matter", "announcement", "font-matter", "fm", "fornt-matter":
// return output, span.Skip{Reason: fmt.Sprintf("suppressed format: %s", article.Type)}
// }
return output, nil
}