From bfde91d5027cbd3d2f622bd485d2871b725200a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D1=8E=D1=84=D1=83=20=D0=98=D0=B2=D0=B0=D0=BD?= Date: Sat, 2 Mar 2024 14:39:37 +1000 Subject: [PATCH 1/2] ready --- otus-18/src/otus_18/homework/pokemons.clj | 99 ++++++++++++++++++++++- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/otus-18/src/otus_18/homework/pokemons.clj b/otus-18/src/otus_18/homework/pokemons.clj index 077ae5b..5987aa6 100644 --- a/otus-18/src/otus_18/homework/pokemons.clj +++ b/otus-18/src/otus_18/homework/pokemons.clj @@ -1,9 +1,16 @@ -(ns otus-18.homework.pokemons) +(ns otus-18.homework.pokemons + (:require [clj-http.client :as client] + [cheshire.core :as cheshire] + [clojure.core.async :as a :refer [! + chan close! + go onto-chan! + thread]])) -(def base-url "https://pokeapi.co/api/v2") +(def base-url "https://pokeapi.co/api/v2") (def pokemons-url (str base-url "/pokemon")) -(def type-path (str base-url "/type")) +(def type-path (str base-url "/type")) +(def n-concurency 5) (defn extract-pokemon-name [pokemon] (:name pokemon)) @@ -13,7 +20,91 @@ (first) :name)) + +(defn async-get [url] + (thread (client/get url))) + +(defn parse-str [s] + (cheshire/parse-string s true)) + +(defn get-parse-xform [url xform] + (->> (client/get url) + :body + parse-str + xform)) + +(defn get-and-parse + "xform may be nil" + [url & xform] + (go (as-> ( (chan 20) + out> (chan 20) + blocking-get-type-name (fn [u] (get-parse-xform u (fn [r] [(:name r) (extract-type-name r lang)])))] + + (a/pipeline-blocking n-concurency out> (map blocking-get-type-name) in>) + + (go (->> ())) + + (let [result ())] + (close! in>) + (close! out>) + result))) + +(defn pokemons-types [count lang] + (let [type-names-lang (translated-types lang) + in> (chan 20) + mdl> (chan count) + out> (chan count) + async-fn (fn [url out*] + (go + (let [arr (get-parse-xform url :results)] + (doseq [x arr] + (>! out* x))) + (close! out*))) + bl-parse-poke (fn [{name :name url :url}] + (let [body-types (get-parse-xform url :types) + type-names (mapv (fn [t] (type-names-lang (get-in t [:type :name]))) body-types)] + {:name name :types type-names}))] + + ; пайплайны + (a/pipeline-async n-concurency mdl> async-fn in>) + (a/pipeline-blocking n-concurency out> (map bl-parse-poke) mdl>) + + ; начало обработки + (go (a/onto-chan! in> (generate-pokemon-urls count count))) + + (let [result ())] + (close! in>) + (close! mdl>) + (close! out>) + result))) + (defn get-pokemons "Асинхронно запрашивает список покемонов и название типов в заданном языке. Возвращает map, где ключами являются имена покемонов (на английском английский), а значения - коллекция названий типов на заданном языке." - [& {:keys [limit lang] :or {limit 50 lang "ja"}}]) + [& {:keys [limit lang] :or {limit 50 lang "ja"}}] + (pokemons-types limit lang)) + +(comment + (pokemons-types 12 "ja")) From 3b1195cd4f98af74a2da7f6366f9dfc368d8669d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D1=8E=D1=84=D1=83=20=D0=98=D0=B2=D0=B0=D0=BD?= Date: Sat, 30 Mar 2024 15:44:35 +1000 Subject: [PATCH 2/2] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=8C=D1=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- otus-18/src/otus_18/homework/pokemons.clj | 81 ++++++++++++----------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/otus-18/src/otus_18/homework/pokemons.clj b/otus-18/src/otus_18/homework/pokemons.clj index 5987aa6..27664a4 100644 --- a/otus-18/src/otus_18/homework/pokemons.clj +++ b/otus-18/src/otus_18/homework/pokemons.clj @@ -1,9 +1,9 @@ (ns otus-18.homework.pokemons - (:require [clj-http.client :as client] - [cheshire.core :as cheshire] + (:require [cheshire.core :as cheshire] + [clj-http.client :as client] [clojure.core.async :as a :refer [! chan close! - go onto-chan! + go onto-chan! thread]])) (def base-url "https://pokeapi.co/api/v2") @@ -37,26 +37,24 @@ "xform may be nil" [url & xform] (go (as-> ( (chan 20) - out> (chan 20) + (map (fn [offset] + (str pokemons-url "?offset=" offset "&limit=" (min batch-size (- total offset)))) + (range 0 total batch-size)) + )) + +(defn translated-types + "Получение переведенных типов" + [lang] + (let [in> (chan) + out> (chan) blocking-get-type-name (fn [u] (get-parse-xform u (fn [r] [(:name r) (extract-type-name r lang)])))] (a/pipeline-blocking n-concurency out> (map blocking-get-type-name) in>) @@ -67,37 +65,40 @@ (onto-chan! in>))) (let [result ())] - (close! in>) - (close! out>) result))) (defn pokemons-types [count lang] - (let [type-names-lang (translated-types lang) - in> (chan 20) - mdl> (chan count) - out> (chan count) - async-fn (fn [url out*] - (go - (let [arr (get-parse-xform url :results)] - (doseq [x arr] - (>! out* x))) - (close! out*))) + (let [type-names-lang (translated-types lang) ; синхронное получение типов 2,5 сек + batch-size 20 + in> (chan) + mdl> (chan) + mdl-2> (chan) + out> (chan) + + ; получаем списки url покемонов (по batch-size) + bl-get-pokes (fn [url] (get-parse-xform url :results)) + + ; можно сказать flatten, pipe массивов в пайп элементов из массивов + async-fn (fn [arr out*] + (go (doseq [x arr] + (>! out* x)) + (close! out*))) + + ; получаем покемона, и ищем его тип в type-names-lang bl-parse-poke (fn [{name :name url :url}] - (let [body-types (get-parse-xform url :types) + (let [ + body-types (get-parse-xform url :types) type-names (mapv (fn [t] (type-names-lang (get-in t [:type :name]))) body-types)] {:name name :types type-names}))] - ; пайплайны - (a/pipeline-async n-concurency mdl> async-fn in>) - (a/pipeline-blocking n-concurency out> (map bl-parse-poke) mdl>) + (a/pipeline-blocking n-concurency mdl> (map bl-get-pokes) in>) + (a/pipeline-async n-concurency mdl-2> async-fn mdl>) + (a/pipeline-blocking n-concurency out> (map bl-parse-poke) mdl-2>) ; начало обработки - (go (a/onto-chan! in> (generate-pokemon-urls count count))) + (go (a/onto-chan! in> (generate-pokemon-urls count batch-size))) ; генерируем урлы (let [result ())] - (close! in>) - (close! mdl>) - (close! out>) result))) (defn get-pokemons @@ -107,4 +108,6 @@ (pokemons-types limit lang)) (comment - (pokemons-types 12 "ja")) + (generate-pokemon-urls 55) + (time (pokemons-types 55 "ja")) + )