diff --git a/otus-18/migrations/001-init.down.sql b/otus-18/migrations/001-init.down.sql new file mode 100644 index 0000000..6c34eb2 --- /dev/null +++ b/otus-18/migrations/001-init.down.sql @@ -0,0 +1,5 @@ +drop table types; +drop table pokemons; +drop table poke_types; +drop table translate_types; + diff --git a/otus-18/migrations/001-init.up.sql b/otus-18/migrations/001-init.up.sql new file mode 100644 index 0000000..06134c0 --- /dev/null +++ b/otus-18/migrations/001-init.up.sql @@ -0,0 +1,24 @@ +create table types +( + id integer primary key, + name varchar(50) -- name eng +); + +create table translate_types +( + id integer, + lang varchar(10), + translate varchar(50) +); + +create table pokemons +( + id integer primary key, + name varchar(50) +); + +create table poke_types +( + id_poke integer, + id_type integer +); diff --git a/otus-18/project.clj b/otus-18/project.clj index 9fa6d89..e9f4891 100644 --- a/otus-18/project.clj +++ b/otus-18/project.clj @@ -4,6 +4,19 @@ :dependencies [[org.clojure/clojure "1.11.1"] [org.clojure/core.async "1.6.673"] + ;; http & json [clj-http "3.12.3"] [clj-http-fake "1.0.4"] - [cheshire "5.11.0"]]) + [cheshire "5.11.0"] + ;; database + [com.h2database/h2 "2.2.224"] + [com.github.seancorfield/next.jdbc "1.3.909"] + [com.github.seancorfield/honeysql "2.5.1103"] + [dev.weavejester/ragtime "0.9.4"] + [com.zaxxer/HikariCP "5.0.1"] + ] + + :main ^:skip-aot otus-18.homework.core + :profiles {:uberjar {:aot :all + :jvm-opts ["-Dclojure.compiler.direct-linking=true"]}} + ) diff --git a/otus-18/src/otus_18/async.clj b/otus-18/src/otus_18/async.clj index 267cbbc..154bcbc 100644 --- a/otus-18/src/otus_18/async.clj +++ b/otus-18/src/otus_18/async.clj @@ -97,7 +97,7 @@ (def ca> (chan 1)) (def cb> (chan 1)) -(defn c-af [val result] ; notice the signature is different for `pipeline-async`, it includes a channel +(defn c-af [val result] ; notice the signature is different for `pipeline-async`, it includes a channel (go (! result (str val "!!!")) (>! result (str val "!!!")) diff --git a/otus-18/src/otus_18/homework/analytical.clj b/otus-18/src/otus_18/homework/analytical.clj new file mode 100644 index 0000000..acfb62a --- /dev/null +++ b/otus-18/src/otus_18/homework/analytical.clj @@ -0,0 +1,66 @@ +(ns otus-18.homework.analytical + (:require + [otus-18.homework.db :refer [execute]] + [otus-18.homework.sql-builder :refer [columns-fn from-fn fun-fn + group-by-fn left-join-fn order-by-fn + to-sql + ]] + )) + +(defn total-pokemons [] + (-> {} + (from-fn :pokemons) + (columns-fn [[(fun-fn :count 1) :cnt]]) + to-sql + execute + (get-in [0 :cnt]))) + +(defn total-types [] + (-> {} + (from-fn :types) + (columns-fn [[(fun-fn :count 1) :cnt]]) + to-sql + execute + (get-in [0 :cnt]))) + + +" +select count(1) as cnt, t.NAME +from POKE_TYPES pt +left join types t on pt.ID_TYPE = t.id +group by ID_TYPE ORDER BY cnt desc +" +(defn popular-types [] + (-> {} + (columns-fn [[(fun-fn :count 1) :cnt] [:t.name]]) + (from-fn [[:poke_types :pt]]) + (left-join-fn [[:types :t] [:= :pt.id_type :t.id]]) + (group-by-fn [:id_type]) + (order-by-fn [[:cnt :desc]]) + to-sql + execute) + ) + +(defn support-languages [] + (let [result (-> {} + (columns-fn [(fun-fn :distinct :lang)]) + (from-fn [:translate_types]) + to-sql + execute + ) + ] + (mapv #(:lang %) result))) + +(defn analytics-print [] + (println "Всего покемонов загружено" (total-pokemons)) + (println "Всего типов покемонов" (total-types)) + (println "Поддерживаемые языки кроме английского" (support-languages)) + (println "Наиболее часто встречаемые типы покемонов" (popular-types)) + ) +(comment + (total-pokemons) + (total-types) + (popular-types) + (support-languages) + (analytics-print) + ) \ No newline at end of file diff --git a/otus-18/src/otus_18/homework/conf/config.clj b/otus-18/src/otus_18/homework/conf/config.clj new file mode 100644 index 0000000..0b42692 --- /dev/null +++ b/otus-18/src/otus_18/homework/conf/config.clj @@ -0,0 +1,5 @@ +(ns otus-18.homework.conf.config) + +(def ctx (atom { + :db {:url "jdbc:h2:file:~/pokemons.h2"} + })) \ No newline at end of file diff --git a/otus-18/src/otus_18/homework/conf/migrations.clj b/otus-18/src/otus_18/homework/conf/migrations.clj new file mode 100644 index 0000000..56443a7 --- /dev/null +++ b/otus-18/src/otus_18/homework/conf/migrations.clj @@ -0,0 +1,23 @@ +(ns otus-18.homework.conf.migrations + (:require [otus-18.homework.conf.config :refer [ctx]] + [ragtime.jdbc :as jdbc] + [ragtime.repl :as repl])) + + +(defn load-config [] + {:datastore (jdbc/sql-database {:connection-uri (get-in @ctx [:db :url])}) + :migrations (jdbc/load-directory "./migrations")}) +; почему-то (jdbc/load-load-resources "migrations") не работает + +(defn migrate [] + (println "migrating ....") + (repl/migrate (load-config)) + (println "migrated")) + +(defn clear [] + (repl/rollback (load-config))) + + +(comment + (migrate) + ) \ No newline at end of file diff --git a/otus-18/src/otus_18/homework/core.clj b/otus-18/src/otus_18/homework/core.clj new file mode 100644 index 0000000..f47bc72 --- /dev/null +++ b/otus-18/src/otus_18/homework/core.clj @@ -0,0 +1,37 @@ +(ns otus-18.homework.core + (:require + [otus-18.homework.analytical :refer [analytics-print]] + [otus-18.homework.conf.migrations :refer [migrate]] + [otus-18.homework.db :refer [save-pokemon save-type]] + [otus-18.homework.pokemons :refer [pokemons translated-types]] + ) + (:gen-class)) +(defn load-types [] + (println "loading types...") + (translated-types "ja" save-type) ; can use println + (println "saved") + ) +(defn load-pokemons [] + (println "loading pokemons...") + (pokemons 55 save-pokemon) ; can use println + (println "saved") + ) + +(defn init [] + (do + (migrate) + (load-types) + (load-pokemons) + )) + + + +(defn -main + [& args] + (do + (init) + (println "Аналитика") + (analytics-print) + (println "Нужно удалить файл ~/pokemons.h2.mv.db") + ) + ) diff --git a/otus-18/src/otus_18/homework/db.clj b/otus-18/src/otus_18/homework/db.clj new file mode 100644 index 0000000..dfabdc4 --- /dev/null +++ b/otus-18/src/otus_18/homework/db.clj @@ -0,0 +1,49 @@ +(ns otus-18.homework.db + (:require [honey.sql :as sql] + [honey.sql.helpers :refer [from insert-into values select values where]] + [next.jdbc :as jdbc] + [next.jdbc.connection :as connection] + [next.jdbc.sql :as jdbc.sql] + [next.jdbc.result-set :as rs] + [otus-18.homework.conf.config :refer [ctx]] + ) + ) + +(def ds (connection/->pool com.zaxxer.hikari.HikariDataSource + {:jdbcUrl (get-in @ctx [:db :url]) + })) + +(defn save-type [map] + (let [ + types-maps (select-keys map [:id :name]) + transl-types-maps (select-keys map [:id :lang :translate]) + ] + (jdbc.sql/insert! ds :types types-maps) + (jdbc.sql/insert! ds :translate_types transl-types-maps) + ) + ;; todo ignore on conflict + ) + +(defn select-type-id-by-name [n] + (-> (select :id) + (from :types) + (where [:= :name n]))) + +(defn save-pokemon [poke] + ;; todo ignore on conflict + (let [ + pokemon (select-keys poke [:id :name]) + id (:id poke) + poke-sql (-> (insert-into :pokemons) ;sql для вставки покемона + (values [pokemon])) + pt-values (mapv (fn [n] {:id_poke id :id_type (select-type-id-by-name n)}) (:types poke)) + pt-sql (-> (insert-into :poke-types) + (values pt-values)) ; sql для вставки в poke-types + ] + (jdbc/execute! ds (sql/format poke-sql)) + (jdbc/execute! ds (sql/format pt-sql)) + ) + ) + +(defn execute [sql] + (jdbc/execute! ds sql {:builder-fn rs/as-unqualified-lower-maps})) \ No newline at end of file diff --git a/otus-18/src/otus_18/homework/pokemons.clj b/otus-18/src/otus_18/homework/pokemons.clj index 077ae5b..78d235e 100644 --- a/otus-18/src/otus_18/homework/pokemons.clj +++ b/otus-18/src/otus_18/homework/pokemons.clj @@ -1,11 +1,16 @@ -(ns otus-18.homework.pokemons) +(ns otus-18.homework.pokemons + (:require [cheshire.core :as cheshire] + [clj-http.client :as client] + [clojure.core.async :as a :refer [! + chan close! + go go-loop onto-chan! + thread]])) -(def base-url "https://pokeapi.co/api/v2") +(def base-url "https://pokeapi.co/api/v2") (def pokemons-url (str base-url "/pokemon")) -(def type-path (str base-url "/type")) +(def type-path (str base-url "/type")) -(defn extract-pokemon-name [pokemon] - (:name pokemon)) +(def n-concurency 5) (defn extract-type-name [pokemon-type lang] (->> (:names pokemon-type) @@ -13,7 +18,102 @@ (first) :name)) -(defn get-pokemons - "Асинхронно запрашивает список покемонов и название типов в заданном языке. Возвращает map, где ключами являются - имена покемонов (на английском английский), а значения - коллекция названий типов на заданном языке." - [& {:keys [limit lang] :or {limit 50 lang "ja"}}]) + +(defn async-get [url] + (thread (client/get url))) + +(defn parse-str [s] + (cheshire/parse-string s true)) + +(defn get-parse-xform [url xform] + (->> (client/get url) + :body + parse-str + xform)) + +(defn get-and-parse + "xform may be nil" + [url & xform] + (go (as-> ( (chan) + out> (chan) + blocking-get-type-name (fn [u] (get-parse-xform u (fn [r] {:name (:name r) :lang lang :id (:id r) :translate (extract-type-name r lang)})))] + + (a/pipeline-blocking n-concurency out> (map blocking-get-type-name) in>) + + (go (->> ())) + + ()] + (save-xform x) + (recur)) + ))) + ) + +(defn pokemons [total save-xform] + "Получение покемонов. + {:id :name :types[:name]} для save-xform" + (let [ + batch-size 20 + in> (chan) + mdl> (chan) + mdl-2> (chan) + out> (chan) + ; получаем списки url покемонов (по batch-size) + bl-get-pokes (fn [url] (get-parse-xform url :results)) + + ; можно сказать flatten, pipe массивов в пайп элементов из массивов + async-fn (fn [arr out*] + (go (doseq [x arr] + (>! out* x)) + (close! out*))) + ; получаем покемона, + bl-parse-poke (fn [{name :name url :url}] + (let [ + body (get-parse-xform url (fn [m] (select-keys m [:types :id]))) + id (:id body) + type-names (mapv (fn [t] (get-in t [:type :name])) (:types body)) + + ] + {:id id :name name :types type-names} + )) + ] + + ; пайплайны + (a/pipeline-blocking n-concurency mdl> (map bl-get-pokes) in>) + (a/pipeline-async n-concurency mdl-2> async-fn mdl>) + (a/pipeline-blocking n-concurency out> (map bl-parse-poke) mdl-2>) + + ; начало обработки + (go (a/onto-chan! in> (generate-pokemon-urls total batch-size))) ; генерируем урлы + + ()] + (save-xform x) + (recur)))) + )) + +(comment + (generate-pokemon-urls 55) + (time (pokemons-types 55 "ja")) + ) diff --git a/otus-18/src/otus_18/homework/sql_builder.clj b/otus-18/src/otus_18/homework/sql_builder.clj new file mode 100644 index 0000000..43ed6f8 --- /dev/null +++ b/otus-18/src/otus_18/homework/sql_builder.clj @@ -0,0 +1,67 @@ +(ns otus-18.homework.sql-builder + (:require [honey.sql :as sql])) + +" +Clojure не java, как упростить даже не представляю. +Все равно ерунда выходит. +" + +" +Такое использование больше похоже на паттерн builder +Но оно ни чем не отличается от самого HoneySql +( -> {} + (from-fn :table-name) + (columns-fn [:name :id]) + (to-sql) +) +" +(defn from-fn [m v] (if (some? v) (assoc m :from v) m)) +(defn columns-fn [m v] (if (some? v) (assoc m :select v) m)) +(defn where-fn [m v] (if (some? v) (assoc m :where v) m)) +(defn left-join-fn [m v] (if (some? v) (assoc m :left-join v) m)) +(defn group-by-fn [m v] (if (some? v) (assoc m :group-by v) m)) +(defn having-fn [m v] (if (some? v) (assoc m :having v) m)) +(defn order-by-fn [m v] (if (some? v) (assoc m :order-by v) m)) +(defn to-sql [m] (sql/format m)) + +(defn fun-fn + "( -> {} + (from-fn :table-name) + (columns-fn [(fun-fn :min :price)]) + (to-sql) + ) + " + [func column] + [[func column]] + ) + + +(defn builder-sql-select + "Еще вариант, но тоже как мне кажется не удобный. + ненужные параметры нужно заполнять nil. + а если принимать мапу с ключами, то это тот же HoneySql + + table и columns обязательны" + [table, columns, where, left-join, group-by, having] + (let [] + (-> {} + (from-fn table) + (columns-fn columns) + (where-fn where) + (left-join-fn left-join) + (group-by-fn group-by) + (having-fn having) + (to-sql) + ) + )) + +(comment + (builder-sql-select :pets [:id :name] [:= :type :dog] nil nil nil) + + (-> {} + (from-fn :table-name) + (columns-fn [(fun-fn :min :price)]) + (to-sql) + ) + + ) \ No newline at end of file diff --git a/otus-18/src/resources/migrations/001-init.down.sql b/otus-18/src/resources/migrations/001-init.down.sql new file mode 100644 index 0000000..6c34eb2 --- /dev/null +++ b/otus-18/src/resources/migrations/001-init.down.sql @@ -0,0 +1,5 @@ +drop table types; +drop table pokemons; +drop table poke_types; +drop table translate_types; + diff --git a/otus-18/src/resources/migrations/001-init.up.sql b/otus-18/src/resources/migrations/001-init.up.sql new file mode 100644 index 0000000..06134c0 --- /dev/null +++ b/otus-18/src/resources/migrations/001-init.up.sql @@ -0,0 +1,24 @@ +create table types +( + id integer primary key, + name varchar(50) -- name eng +); + +create table translate_types +( + id integer, + lang varchar(10), + translate varchar(50) +); + +create table pokemons +( + id integer primary key, + name varchar(50) +); + +create table poke_types +( + id_poke integer, + id_type integer +);