Isyufu/HW 27 Базы данных

otus-18/migrations/001-init.down.sql

@@ -0,0 +1,5 @@
+drop table types;
+drop table pokemons;
+drop table poke_types;
+drop table translate_types;
+

otus-18/migrations/001-init.up.sql

@@ -0,0 +1,24 @@
+create table types
+(
+    id   integer primary key,
+    name varchar(50) -- name eng
+);
+
+create table translate_types
+(
+    id   integer,
+    lang      varchar(10),
+    translate varchar(50)
+);
+
+create table pokemons
+(
+    id   integer primary key,
+    name varchar(50)
+);
+
+create table poke_types
+(
+    id_poke integer,
+    id_type integer
+);

otus-18/project.clj

@@ -4,6 +4,19 @@
 
   :dependencies [[org.clojure/clojure "1.11.1"]
                  [org.clojure/core.async "1.6.673"]
+                 ;; http & json
                  [clj-http "3.12.3"]
                  [clj-http-fake "1.0.4"]
-                 [cheshire "5.11.0"]])
+                 [cheshire "5.11.0"]
+                 ;; database
+                 [com.h2database/h2 "2.2.224"]
+                 [com.github.seancorfield/next.jdbc "1.3.909"]
+                 [com.github.seancorfield/honeysql "2.5.1103"]
+                 [dev.weavejester/ragtime "0.9.4"]
+                 [com.zaxxer/HikariCP  "5.0.1"]
+                 ]
+
+  :main ^:skip-aot otus-18.homework.core
+  :profiles {:uberjar {:aot :all
+                       :jvm-opts ["-Dclojure.compiler.direct-linking=true"]}}
+  )

otus-18/src/otus_18/async.clj

@@ -97,7 +97,7 @@
 (def ca> (chan 1))
 (def cb> (chan 1))
 
-(defn c-af [val result] ; notice the signature is different for `pipeline-async`, it includes a channel
+(defn c-af [val result]                                     ; notice the signature is different for `pipeline-async`, it includes a channel
   (go (<! (timeout 1000))
       (>! result (str val "!!!"))
       (>! result (str val "!!!"))

otus-18/src/otus_18/homework/analytical.clj

@@ -0,0 +1,66 @@
+(ns otus-18.homework.analytical
+  (:require
+    [otus-18.homework.db :refer [execute]]
+    [otus-18.homework.sql-builder :refer [columns-fn from-fn fun-fn
+                                          group-by-fn left-join-fn order-by-fn
+                                          to-sql
+                                          ]]
+    ))
+
+(defn total-pokemons []
+  (-> {}
+      (from-fn :pokemons)
+      (columns-fn [[(fun-fn :count 1) :cnt]])
+      to-sql
+      execute
+      (get-in [0 :cnt])))
+
+(defn total-types []
+  (-> {}
+      (from-fn :types)
+      (columns-fn [[(fun-fn :count 1) :cnt]])
+      to-sql
+      execute
+      (get-in [0 :cnt])))
+
+
+"
+select count(1) as cnt, t.NAME
+from POKE_TYPES pt
+left join types t on pt.ID_TYPE = t.id
+group by ID_TYPE ORDER BY cnt desc
+"
+(defn popular-types []
+  (-> {}
+      (columns-fn [[(fun-fn :count 1) :cnt] [:t.name]])
+      (from-fn [[:poke_types :pt]])
+      (left-join-fn [[:types :t] [:= :pt.id_type :t.id]])
+      (group-by-fn [:id_type])
+      (order-by-fn [[:cnt :desc]])
+      to-sql
+      execute)
+  )
+
+(defn support-languages []
+  (let [result (-> {}
+                   (columns-fn [(fun-fn :distinct :lang)])
+                   (from-fn [:translate_types])
+                   to-sql
+                   execute
+                   )
+        ]
+    (mapv #(:lang %) result)))
+
+(defn analytics-print []
+  (println "Всего покемонов загружено" (total-pokemons))
+  (println "Всего типов покемонов" (total-types))
+  (println "Поддерживаемые языки кроме английского" (support-languages))
+  (println "Наиболее часто встречаемые типы покемонов" (popular-types))
+  )
+(comment
+  (total-pokemons)
+  (total-types)
+  (popular-types)
+  (support-languages)
+  (analytics-print)
+  )

otus-18/src/otus_18/homework/conf/config.clj

@@ -0,0 +1,5 @@
+(ns otus-18.homework.conf.config)
+
+(def ctx (atom {
+                :db {:url "jdbc:h2:file:~/pokemons.h2"}
+                }))

otus-18/src/otus_18/homework/conf/migrations.clj

@@ -0,0 +1,23 @@
+(ns otus-18.homework.conf.migrations
+  (:require [otus-18.homework.conf.config :refer [ctx]]
+            [ragtime.jdbc :as jdbc]
+            [ragtime.repl :as repl]))
+
+
+(defn load-config []
+  {:datastore  (jdbc/sql-database {:connection-uri (get-in @ctx [:db :url])})
+   :migrations (jdbc/load-directory "./migrations")})
+; почему-то (jdbc/load-load-resources "migrations") не работает
+
+(defn migrate []
+  (println "migrating ....")
+  (repl/migrate (load-config))
+  (println "migrated"))
+
+(defn clear []
+  (repl/rollback (load-config)))
+
+
+(comment
+  (migrate)
+  )

otus-18/src/otus_18/homework/core.clj

@@ -0,0 +1,37 @@
+(ns otus-18.homework.core
+  (:require
+    [otus-18.homework.analytical :refer [analytics-print]]
+    [otus-18.homework.conf.migrations :refer [migrate]]
+    [otus-18.homework.db :refer [save-pokemon save-type]]
+    [otus-18.homework.pokemons :refer [pokemons translated-types]]
+    )
+  (:gen-class))
+(defn load-types []
+  (println "loading types...")
+  (translated-types "ja" save-type)                         ; can use println
+  (println "saved")
+  )
+(defn load-pokemons []
+  (println "loading pokemons...")
+  (pokemons 55 save-pokemon)                                ; can use println
+  (println "saved")
+  )
+
+(defn init []
+  (do
+    (migrate)
+    (load-types)
+    (load-pokemons)
+    ))
+
+
+
+(defn -main
+  [& args]
+  (do
+    (init)
+    (println "Аналитика")
+    (analytics-print)
+    (println "Нужно удалить файл ~/pokemons.h2.mv.db")
+    )
+  )

otus-18/src/otus_18/homework/db.clj

@@ -0,0 +1,49 @@
+(ns otus-18.homework.db
+  (:require [honey.sql :as sql]
+            [honey.sql.helpers :refer [from insert-into values select values where]]
+            [next.jdbc :as jdbc]
+            [next.jdbc.connection :as connection]
+            [next.jdbc.sql :as jdbc.sql]
+            [next.jdbc.result-set :as rs]
+            [otus-18.homework.conf.config :refer [ctx]]
+            )
+  )
+
+(def ds (connection/->pool com.zaxxer.hikari.HikariDataSource
+                           {:jdbcUrl (get-in @ctx [:db :url])
+                            }))
+
+(defn save-type [map]
+  (let [
+        types-maps (select-keys map [:id :name])
+        transl-types-maps (select-keys map [:id :lang :translate])
+        ]
+    (jdbc.sql/insert! ds :types types-maps)
+    (jdbc.sql/insert! ds :translate_types transl-types-maps)
+    )
+  ;; todo ignore on conflict
+  )
+
+(defn select-type-id-by-name [n]
+  (-> (select :id)
+      (from :types)
+      (where [:= :name n])))
+
+(defn save-pokemon [poke]
+  ;; todo ignore on conflict
+  (let [
+        pokemon (select-keys poke [:id :name])
+        id (:id poke)
+        poke-sql (-> (insert-into :pokemons)                ;sql для вставки покемона
+                     (values [pokemon]))
+        pt-values (mapv (fn [n] {:id_poke id :id_type (select-type-id-by-name n)}) (:types poke))
+        pt-sql (-> (insert-into :poke-types)
+                   (values pt-values))                      ; sql для вставки в poke-types
+        ]
+    (jdbc/execute! ds (sql/format poke-sql))
+    (jdbc/execute! ds (sql/format pt-sql))
+    )
+  )
+
+(defn execute [sql]
+  (jdbc/execute! ds sql {:builder-fn rs/as-unqualified-lower-maps}))

otus-18/src/otus_18/homework/pokemons.clj

@@ -1,19 +1,119 @@
-(ns otus-18.homework.pokemons)
+(ns otus-18.homework.pokemons
+  (:require [cheshire.core :as cheshire]
+            [clj-http.client :as client]
+            [clojure.core.async :as a :refer [<! <!! >!
+                                              chan close!
+                                              go go-loop onto-chan!
+                                              thread]]))
 
-(def base-url     "https://pokeapi.co/api/v2")
+(def base-url "https://pokeapi.co/api/v2")
 (def pokemons-url (str base-url "/pokemon"))
-(def type-path    (str base-url "/type"))
+(def type-path (str base-url "/type"))
 
-(defn extract-pokemon-name [pokemon]
-  (:name pokemon))
+(def n-concurency 5)
 
 (defn extract-type-name [pokemon-type lang]
   (->> (:names pokemon-type)
        (filter (fn [type-name] (= lang (-> type-name :language :name))))
        (first)
        :name))
 
-(defn get-pokemons
-  "Асинхронно запрашивает список покемонов и название типов в заданном языке. Возвращает map, где ключами являются
-  имена покемонов (на английском английский), а значения - коллекция названий типов на заданном языке."
-  [& {:keys [limit lang] :or {limit 50 lang "ja"}}])
+
+(defn async-get [url]
+  (thread (client/get url)))
+
+(defn parse-str [s]
+  (cheshire/parse-string s true))
+
+(defn get-parse-xform [url xform]
+  (->> (client/get url)
+       :body
+       parse-str
+       xform))
+
+(defn get-and-parse
+  "xform may be nil"
+  [url & xform]
+  (go (as-> (<! (async-get url)) m
+            (:body m)
+            (cheshire/parse-string m true)
+            (if (some? xform) ((first xform) m) m))))
+
+; генерируем урлы получения списка покемонов, например по 20 штук
+(defn generate-pokemon-urls
+  ([total] (generate-pokemon-urls total 20))
+  ([total batch-size]
+   (map (fn [offset]
+          (str pokemons-url "?offset=" offset "&limit=" (min batch-size (- total offset))))
+        (range 0 total batch-size))
+   ))
+
+(defn translated-types
+  "Получение переведенных типов.
+  {:id :name :lang :translate} для save-xform"
+  [lang save-xform]
+  (let [in> (chan)
+        out> (chan)
+        blocking-get-type-name (fn [u] (get-parse-xform u (fn [r] {:name (:name r) :lang lang :id (:id r) :translate (extract-type-name r lang)})))]
+
+    (a/pipeline-blocking n-concurency out> (map blocking-get-type-name) in>)
+
+    (go (->> (<! (get-and-parse type-path))
+             :results
+             (map :url)
+             (onto-chan! in>)))
+
+    (<!! (go-loop []
+           (when-let [x (<! out>)]
+             (save-xform x)
+             (recur))
+           )))
+  )
+
+(defn pokemons [total save-xform]
+  "Получение покемонов.
+   {:id :name :types[:name]} для save-xform"
+  (let [
+        batch-size 20
+        in> (chan)
+        mdl> (chan)
+        mdl-2> (chan)
+        out> (chan)
+        ; получаем списки url покемонов (по batch-size)
+        bl-get-pokes (fn [url] (get-parse-xform url :results))
+
+        ; можно сказать flatten, pipe массивов в пайп элементов из массивов
+        async-fn (fn [arr out*]
+                   (go (doseq [x arr]
+                         (>! out* x))
+                       (close! out*)))
+        ; получаем покемона,
+        bl-parse-poke (fn [{name :name url :url}]
+                        (let [
+                              body (get-parse-xform url (fn [m] (select-keys m [:types :id])))
+                              id (:id body)
+                              type-names (mapv (fn [t] (get-in t [:type :name])) (:types body))
+
+                              ]
+                          {:id id :name name :types type-names}
+                          ))
+        ]
+
+    ; пайплайны
+    (a/pipeline-blocking n-concurency mdl> (map bl-get-pokes) in>)
+    (a/pipeline-async n-concurency mdl-2> async-fn mdl>)
+    (a/pipeline-blocking n-concurency out> (map bl-parse-poke) mdl-2>)
+
+    ; начало обработки
+    (go (a/onto-chan! in> (generate-pokemon-urls total batch-size))) ; генерируем урлы
+
+    (<!! (go-loop []
+           (when-let [x (<! out>)]
+             (save-xform x)
+             (recur))))
+    ))
+
+(comment
+  (generate-pokemon-urls 55)
+  (time (pokemons-types 55 "ja"))
+  )