-
-
Notifications
You must be signed in to change notification settings - Fork 3
/
pypi.clj
210 lines (189 loc) · 8.36 KB
/
pypi.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
;; Copyright © 2020-2022 Vitaly Samigullin
;;
;; This program and the accompanying materials are made available under the
;; terms of the Eclipse Public License 2.0 which is available at
;; http://www.eclipse.org/legal/epl-2.0.
;;
;; This Source Code may also be made available under the following Secondary
;; Licenses when the conditions for such availability set forth in the Eclipse
;; Public License, v. 2.0 are satisfied: GNU General Public License as published by
;; the Free Software Foundation, either version 2 of the License, or (at your
;; option) any later version, with the GNU Classpath Exception which is available
;; at https://www.gnu.org/software/classpath/license.html.
;;
;; SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
(ns pip-license-checker.pypi
"Python PyPI API functions"
(:gen-class)
(:require
[cheshire.core :as json]
[clojure.string :as str]
[indole.core :refer [make-rate-limiter]]
[pip-license-checker.data :as d]
[pip-license-checker.exception :as exception]
[pip-license-checker.file :as file]
[pip-license-checker.filters :as filters]
[pip-license-checker.github :as github]
[pip-license-checker.http :as http]
[pip-license-checker.license :as license]
[pip-license-checker.version :as version]))
(def settings-http-client
{:socket-timeout 3000
:connection-timeout 3000
:max-redirects 3})
(def url-pypi-base "https://pypi.org/pypi")
(def license-undefined #{"" "UNKNOWN" [] ["UNKNOWN"]})
(def unspecific-license-classifiers #{"License :: OSI Approved"})
(def regex-match-classifier #"License :: .*")
(def regex-split-classifier #" :: ")
(def regex-split-specifier-ops #"(===|==|~=|!=|>=|<=|<|>)")
(def req-status-found :found)
(def req-status-error :error)
;; Get API response, parse it
(defn api-request-releases
"Moved out as a standalone function for testing simplicity"
[url rate-limiter]
(http/request-get url settings-http-client rate-limiter))
(defn api-get-releases
"Get seq of versions available for a package
NB! versions are not sorted!"
[package-name rate-limiter]
(let [url (str/join "/" [url-pypi-base package-name "json"])
resp (try (api-request-releases url rate-limiter)
(catch Exception e e))
error (when (instance? Exception resp)
(exception/get-error-message "PyPI::releases" resp))
data (when (nil? error) resp)
versions (-> data
:body
json/parse-string
(get "releases")
keys)
releases (->> versions
(map #(version/parse-version %))
(filter #(not (nil? %))))]
releases))
(defn api-request-project
"Moved out as a standalone function for testing simplicity"
[url rate-limiter]
(http/request-get url settings-http-client rate-limiter))
(defn api-get-project
"Return respone of GET request to PyPI API for requirement"
[requirement options rate-limiter]
(let [{:keys [name specifiers]} requirement
releases (api-get-releases name rate-limiter)
version (version/get-version specifiers releases :pre (:pre options))
url
(if (nil? version)
(str/join "/" [url-pypi-base name "json"])
(str/join "/" [url-pypi-base name version "json"]))
resp (try
(api-request-project url rate-limiter)
(catch Exception e e))
error (when (instance? Exception resp)
(exception/get-error-message "PyPI::project" resp))
resp-data (when (nil? error) resp)
requirement-with-version
(d/map->Requirement {:name name
:version (or version (:orig (last (first specifiers))))
:specifiers specifiers})]
(cond
(and resp-data version)
(d/map->PyPiProject {:status req-status-found
:requirement requirement-with-version
:api-response
(->
resp-data
:body
json/parse-string)
:license nil
:error error})
error
(d/map->PyPiProject {:status req-status-error
:requirement requirement-with-version
:api-response nil
:license (license/get-license-error nil)
:error error})
(nil? version)
(d/map->PyPiProject {:status req-status-error
:requirement requirement-with-version
:api-response nil
:license (license/get-license-error nil)
:error (format "PyPI::version Not found")}))))
;; Helpers to get license name and description
(defn classifiers->license
"Get first most detailed license name from PyPI trove classifiers list"
[classifiers]
(let [classifier (->> classifiers
(filter #(re-matches regex-match-classifier %))
(remove #(contains? unspecific-license-classifiers %))
(map #(last (str/split % regex-split-classifier)))
(str/join ", "))
result (if (= classifier "") nil classifier)]
result))
(defn api-response->license-map
"Get license name from info.classifiers or info.license field of PyPI API data"
[api-response options rate-limiter]
(let [info (get api-response "info")
{:strs [license classifiers home_page]} info
license-license (if (contains? license-undefined license) nil license)
classifiers-license (classifiers->license classifiers)
name (or
classifiers-license
license-license)
gh-license (when (nil? name) (github/homepage->license home_page options rate-limiter))
gh-error (:error gh-license)
license-name (or name (:name gh-license))
license (license/license-with-type license-name)
error-chain (exception/join-ex-info (:error license) gh-error)
result (d/->License (:name license) (:type license) error-chain)]
result))
;; Get license data from API JSON
(defn requirement->rec
"Parse requirement string into map with package name and its specifiers parsed"
[requirement-line]
(let [package-name (-> requirement-line
(str/split regex-split-specifier-ops)
first)
specifiers-str (subs requirement-line (count package-name))
specifiers-vec (version/parse-specifiers specifiers-str)
specifiers (if (= specifiers-vec [nil]) nil specifiers-vec)
result (d/->Requirement package-name nil specifiers)]
result))
(defn requirement->dep
"Return dependency object"
[requirement-rec options rate-limiter]
(let [resp-data (api-get-project requirement-rec options rate-limiter)
{:keys [status requirement api-response error]} resp-data
license (if (= status req-status-error)
(d/->License license/name-error license/type-error nil)
(api-response->license-map api-response options rate-limiter))
error-chain (exception/join-ex-info error (:error license))
project (d/map->Dependency
{:requirement requirement
:license license
:error error-chain})]
project))
(defn get-all-requirements
"Get a sequence of all requirements"
[packages requirements]
(let [file-packages (file/get-requirement-lines requirements)]
(concat packages file-packages)))
;; Entrypoint
(defn get-parsed-deps
"Apply filters and get verdicts for all deps"
[packages requirements options]
(let [exclude-pattern (:exclude options)
rate-limiter (make-rate-limiter
(or (get-in options [:rate-limits :millis]) 60000)
(or (get-in options [:rate-limits :requests]) 120))
licenses (->> (get-all-requirements packages requirements)
(filters/remove-requirements-internal-rules)
(filters/remove-requirements-user-rules exclude-pattern)
(map filters/sanitize-requirement)
(map requirement->rec)
(pmap #(requirement->dep
%
options
rate-limiter)))]
licenses))