From 626a38908f646e8452b255f01e99bd8ca14be75f Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Tue, 4 Jul 2023 19:18:59 -0300 Subject: [PATCH 01/20] atualiza deps e cria app de crawler inicial --- apps/crawler/.formatter.exs | 4 ++++ apps/crawler/.gitignore | 26 ++++++++++++++++++++++++ apps/crawler/README.md | 3 +++ apps/crawler/mix.exs | 33 +++++++++++++++++++++++++++++++ apps/crawler/test/test_helper.exs | 1 + mix.lock | 30 ++++++++++++++++++++++------ 6 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 apps/crawler/.formatter.exs create mode 100644 apps/crawler/.gitignore create mode 100644 apps/crawler/README.md create mode 100644 apps/crawler/mix.exs create mode 100644 apps/crawler/test/test_helper.exs diff --git a/apps/crawler/.formatter.exs b/apps/crawler/.formatter.exs new file mode 100644 index 00000000..d2cda26e --- /dev/null +++ b/apps/crawler/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/apps/crawler/.gitignore b/apps/crawler/.gitignore new file mode 100644 index 00000000..98390be6 --- /dev/null +++ b/apps/crawler/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +crawler-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/apps/crawler/README.md b/apps/crawler/README.md new file mode 100644 index 00000000..e77b286d --- /dev/null +++ b/apps/crawler/README.md @@ -0,0 +1,3 @@ +# Crawler + +**TODO: Add description** diff --git a/apps/crawler/mix.exs b/apps/crawler/mix.exs new file mode 100644 index 00000000..9bedf547 --- /dev/null +++ b/apps/crawler/mix.exs @@ -0,0 +1,33 @@ +defmodule Crawler.MixProject do + use Mix.Project + + def project do + [ + app: :crawler, + version: "0.1.0", + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:crawly, "~> 0.15.0"}, + {:floki, "~> 0.34.0"}, + {:explorer, "~> 0.5.0"} + ] + end +end diff --git a/apps/crawler/test/test_helper.exs b/apps/crawler/test/test_helper.exs new file mode 100644 index 00000000..869559e7 --- /dev/null +++ b/apps/crawler/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/mix.lock b/mix.lock index 52ec8b7f..7e25a579 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,5 @@ %{ - "absinthe": {:hex, :absinthe, "1.7.1", "aca6f64994f0914628429ddbdfbf24212747b51780dae189dd98909da911757b", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c0c4dbd93881fa3bfbad255608234b104b877c2a901850c1fe8c53b408a72a57"}, + "absinthe": {:hex, :absinthe, "1.7.3", "128f9de8d8feab761a50483011c2652074de0a670316d0e24a4979daeb460c8f", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6def91514f023832dbb3433baa166366881648932211f2e8146f9792b08b7bb3"}, "absinthe_phoenix": {:hex, :absinthe_phoenix, "2.0.2", "e607b438db900049b9b3760f8ecd0591017a46122fffed7057bf6989020992b5", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:absinthe_plug, "~> 1.5", [hex: :absinthe_plug, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.5", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.13 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "d36918925c380dc7d2ed7d039c9a3b4182ec36723f7417a68745ade5aab22f8d"}, "absinthe_plug": {:hex, :absinthe_plug, "1.5.8", "38d230641ba9dca8f72f1fed2dfc8abd53b3907d1996363da32434ab6ee5d6ab", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bbb04176647b735828861e7b2705465e53e2cf54ccf5a73ddd1ebd855f996e5a"}, "bandit": {:hex, :bandit, "0.7.7", "48456d09022607a312cf723a91992236aeaffe4af50615e6e2d2e383fb6bef10", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 0.6.7", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "772f0a32632c2ce41026d85e24b13a469151bb8cea1891e597fb38fde103640a"}, @@ -10,25 +10,35 @@ "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"}, + "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, + "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, + "crawly": {:hex, :crawly, "0.15.0", "dd8588d65c808ef96c7df120b9d5d832632336dbf217642c87d6156b5a39cf46", [:mix], [{:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:ex_json_schema, "~> 0.9.2", [hex: :ex_json_schema, repo: "hexpm", optional: false]}, {:gollum, "~> 0.4.0", [hex: :new_gollum, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:poison, "~> 3.1", [hex: :poison, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.9", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "2f53162b6f0de8e5d2fc2532b5b83a5dc944ed00d1fa8515924c77957af4bba4"}, "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, - "dart_sass": {:hex, :dart_sass, "0.6.0", "1fe560c3ed5c577b6b9cf97134a0e05c82b69645d313b1ef0ffb4d659c3d0300", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "41f7bb065b5c30c3ea05e8b41aa3f9b5c62817079b94f70e2a22d133828475bb"}, + "dart_sass": {:hex, :dart_sass, "0.7.0", "7979e056cb74fd6843e1c72db763cffc7726a9192a657735b7d24c0d9c26a1ce", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "4a8e70bca41aa00846398abdf5ad8a64d7907a0f7bf40145cd2e40d5971629f2"}, "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, "earmark": {:hex, :earmark, "1.4.38", "ba8fda946c259c6e8f6759d3647d448e9216e2c0afed8c6ae7f8ce1f7072a497", [:mix], [{:earmark_parser, "~> 1.4.32", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "f938e30de4167e7d8f3bf588b01dc041138278dda1e5a13fb9ec89b43dd5ec7f"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, "ecto": {:hex, :ecto, "3.10.2", "6b887160281a61aa16843e47735b8a266caa437f80588c3ab80a8a960e6abe37", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6a895778f0d7648a4b34b486af59a1c8009041fbdf2b17f1ac215eb829c60235"}, "ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"}, "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, + "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "esbuild": {:hex, :esbuild, "0.7.0", "ce3afb13cd2c5fd63e13c0e2d0e0831487a97a7696cfa563707342bb825d122a", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "4ae9f4f237c5ebcb001390b8ada65a12fb2bb04f3fe3d1f1692b7a06fbfe8752"}, + "esbuild": {:hex, :esbuild, "0.7.1", "fa0947e8c3c3c2f86c9bf7e791a0a385007ccd42b86885e8e893bdb6631f5169", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "66661cdf70b1378ee4dc16573fcee67750b59761b2605a0207c267ab9d19f13c"}, + "ex_json_schema": {:hex, :ex_json_schema, "0.9.3", "fc17c50d410fd99fa6e814e1aed60122d8ff2578b869d17a9db1ce1c621382b6", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "b79962d09cefd33001706255187bdb483a0c2b4442d5edc6822eb7574a8df0a8"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, + "explorer": {:hex, :explorer, "0.5.7", "dd8d3dee6004ef18ca4cfaf9683f069b0dbc5fb84b758a3458b7dd8330ea88e5", [:mix], [{:nx, "~> 0.4.0 or ~> 0.5.0", [hex: :nx, repo: "hexpm", optional: true]}, {:rustler, "~> 0.28.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.5", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}, {:table, "~> 0.1.2", [hex: :table, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "b7e23182c72fc5d2c523ff55f54276719667c3257a00e25daea2f0e01bdea522"}, "expo": {:hex, :expo, "0.4.1", "1c61d18a5df197dfda38861673d392e642649a9cef7694d2f97a587b2cfb319b", [:mix], [], "hexpm", "2ff7ba7a798c8c543c12550fa0e2cbc81b95d4974c65855d8d15ba7b37a1ce47"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.34.3", "5e2dcaec5d7c228ce5b1d3501502e308b2d79eb655e4191751a1fe491c37feac", [:mix], [], "hexpm", "9577440eea5b97924b4bf3c7ea55f7b8b6dce589f9b28b096cc294a8dc342341"}, - "gettext": {:hex, :gettext, "0.22.2", "6bfca374de34ecc913a28ba391ca184d88d77810a3e427afa8454a71a51341ac", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "8a2d389673aea82d7eae387e6a2ccc12660610080ae7beb19452cfdc1ec30f60"}, + "gettext": {:hex, :gettext, "0.22.3", "c8273e78db4a0bb6fba7e9f0fd881112f349a3117f7f7c598fa18c66c888e524", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "935f23447713954a6866f1bb28c3a878c4c011e802bcd68a726f5e558e4b64bd"}, + "gollum": {:hex, :new_gollum, "0.4.0", "89e3e2fc5abd032455341c4a03bcef7042b8d08e02c51df24b99a1a0a1ad69b1", [:mix], [{:httpoison, "~> 1.7", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "85c68465e8678637638656945677062a4e7086e91a04d5c4bca1027321c74582"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~> 2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, + "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "lucide_icons": {:hex, :lucide_icons, "1.0.0", "b3609534b60a36c2018d462e40b3bfbcde4bf5f7d3b02f531319b3a3a72a139e", [:mix], [{:phoenix_html, "~> 3.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.17", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}], "hexpm", "7feb3b7aaa53e85357d6afbd12d96a9d2d5fbe5cb224f63f663ac3a1ac6dca34"}, @@ -52,10 +62,16 @@ "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"}, "phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, + "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "postgrex": {:hex, :postgrex, "0.17.1", "01c29fd1205940ee55f7addb8f1dc25618ca63a8817e56fac4f6846fc2cddcbe", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "14b057b488e73be2beee508fb1955d8db90d6485c6466428fe9ccf1d6692a555"}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.6.1", "160b545bce8bf9a3f1b436b2c10f53574036a0db628e40f393328cbbe593602f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "0dd269fa261c4e3df290b12031c575fff07a542749f7b0e8b744d72d66c43600"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "tailwind": {:hex, :tailwind, "0.2.0", "95f9e4a32020c5bec480f1d6a43a49ac8030b13183127b577605f506d6e13a66", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "385e939fcd7fe4654be5130b187e358aaabade385513f9d200ffecdbb9552a9e"}, + "table": {:hex, :table, "0.1.2", "87ad1125f5b70c5dea0307aa633194083eb5182ec537efc94e96af08937e14a8", [:mix], [], "hexpm", "7e99bc7efef806315c7e65640724bf165c3061cdc5d854060f74468367065029"}, + "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"}, + "tailwind": {:hex, :tailwind, "0.2.1", "83d8eadbe71a8e8f67861fe7f8d51658ecfb258387123afe4d9dc194eddc36b0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "e8a13f6107c95f73e58ed1b4221744e1eb5a093cd1da244432067e19c8c9a277"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"}, @@ -65,4 +81,6 @@ "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "websock": {:hex, :websock, "0.5.2", "b3c08511d8d79ed2c2f589ff430bd1fe799bb389686dafce86d28801783d8351", [:mix], [], "hexpm", "925f5de22fca6813dfa980fb62fd542ec43a2d1a1f83d2caec907483fe66ff05"}, "websock_adapter": {:hex, :websock_adapter, "0.5.3", "4908718e42e4a548fc20e00e70848620a92f11f7a6add8cf0886c4232267498d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "cbe5b814c1f86b6ea002b52dd99f345aeecf1a1a6964e209d208fb404d930d3d"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, } From 15b95ed5804d8b40acd6fa69baccb4ea4ce340e3 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:49:05 -0300 Subject: [PATCH 02/20] =?UTF-8?q?feat:=20adiciona=20reposit=C3=B3rio=20par?= =?UTF-8?q?a=20app=20cotacoes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/cotacoes/i_manage_repository.ex | 19 ++++++ apps/cotacoes/lib/cotacoes/repository.ex | 64 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 apps/cotacoes/lib/cotacoes/i_manage_repository.ex create mode 100644 apps/cotacoes/lib/cotacoes/repository.ex diff --git a/apps/cotacoes/lib/cotacoes/i_manage_repository.ex b/apps/cotacoes/lib/cotacoes/i_manage_repository.ex new file mode 100644 index 00000000..526931a2 --- /dev/null +++ b/apps/cotacoes/lib/cotacoes/i_manage_repository.ex @@ -0,0 +1,19 @@ +defmodule Cotacoes.IManageRepository do + alias Cotacoes.Models.Cotacao + alias Cotacoes.Models.CotacaoPescado + alias Cotacoes.Models.Pescado + + @opaque changeset :: Ecto.Changeset.t() + + @callback find_all_cotacao_by_is_ingested :: list(Cotacao.t()) + @callback insert_all_cotacao(list(Cotacao.t())) :: {:ok, list(Cotacao.t()) | nil} + @callback list_cotacao :: list(Cotacao.t()) + @callback update_all_cotacao(list(Cotacao.t())) :: {:ok, list(Cotacao.t()) | nil} + @callback upsert_cotacao(map) :: {:ok, Cotacao.t()} | {:error, changeset} + + @callback upsert_cotacao_pescado(map) :: {:ok, CotacaoPescado.t()} | {:error, changeset} + + @callback insert_all_pescado(list(Pescadot.t())) :: {:ok, list(Pescado.t()) | nil} + @callback list_pescado :: list(Pescado.t()) + @callback upsert_pescado(map) :: {:ok, Pescado.t()} | {:error, changeset} +end diff --git a/apps/cotacoes/lib/cotacoes/repository.ex b/apps/cotacoes/lib/cotacoes/repository.ex new file mode 100644 index 00000000..1be6d224 --- /dev/null +++ b/apps/cotacoes/lib/cotacoes/repository.ex @@ -0,0 +1,64 @@ +defmodule Cotacoes.Repository do + use Database, :repository + + alias Cotacoes.Models.Cotacao + alias Cotacoes.Models.CotacaoPescado + alias Cotacoes.Models.Pescado + + @behaviour Cotacoes.IManageRepository + + @impl true + def find_all_cotacao_by_is_ingested do + query = from c in Cotacao, where: c.ingested?, select: c + Repo.Replica.all(query) + end + + @impl true + def insert_all_cotacao(cotacoes) do + {_amount, inserted} = Repo.insert_all(Cotacao, cotacoes) + {:ok, inserted} + end + + @impl true + def list_cotacao do + Repo.Replica.all(Cotacao) + end + + @impl true + def update_all_cotacao(cotacoes) do + {_amount, updated} = Repo.update_all(Cotacao, cotacoes) + {:ok, updated} + end + + @impl true + def upsert_cotacao(cotacao \\ %Cotacao{}, attrs) do + cotacao + |> Cotacao.changeset(attrs) + |> Repo.insert_or_update() + end + + @impl true + def upsert_cotacao_pescado(cot_pescado \\ %CotacaoPescado{}, attrs) do + cot_pescado + |> CotacaoPescado.changeset(attrs) + |> Repo.insert_or_update() + end + + @impl true + def insert_all_pescado(pescados) do + {_amount, inserted} = Repo.insert_all(Pescado, pescados) + {:ok, inserted} + end + + @impl true + def list_pescado do + Repo.Replica.all(Pescado) + end + + @impl true + def upsert_pescado(pescado \\ %Pescado{}, attrs) do + pescado + |> Pescado.changeset(attrs) + |> Repo.insert_or_update() + end +end From 21c588e32c5980a8aa3eba312f00965306cbeaef Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:49:33 -0300 Subject: [PATCH 03/20] feat: adiciona handlers para entidade cotacao --- .../lib/cotacoes/handlers/cotacao_handler.ex | 42 +++++++++++++++++++ .../handlers/i_manage_cotacao_handler.ex | 8 ++++ 2 files changed, 50 insertions(+) create mode 100644 apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex create mode 100644 apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex diff --git a/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex new file mode 100644 index 00000000..3b406488 --- /dev/null +++ b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex @@ -0,0 +1,42 @@ +defmodule Cotacoes.Handlers.CotacaoHandler do + alias Cotacoes.Repository + + @behaviour Cotacoes.Handlers.IManageCotacaoHandler + + @impl true + def find_cotacoes_not_ingested do + Repository.find_all_cotacao_by_is_ingested() + end + + @impl true + def ingest_cotacoes(cotacoes) do + {:ok, _} = Repository.update_all_cotacao(cotacoes) + :ok + end + + @impl true + def insert_cotacoes!(cotacoes) do + Repository.insert_all_cotacao(cotacoes) + end + + @impl true + def reject_inserted_cotacoes(cotacoes) do + cotacoes_ids = + cotacoes + |> Enum.sort_by(& &1.id_publico) + |> Enum.map(& &1.link) + |> MapSet.new() + + current = Repository.list_cotacao() + + current_ids = + current + |> Enum.sort_by(& &1.id_publico) + |> Enum.map(& &1.link) + |> MapSet.new() + + diff_ids = MapSet.difference(current_ids, cotacoes_ids) + + Enum.filter(cotacoes ++ current, &(&1 in diff_ids)) + end +end diff --git a/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex new file mode 100644 index 00000000..3c477229 --- /dev/null +++ b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex @@ -0,0 +1,8 @@ +defmodule Cotacoes.Handlers.IManageCotacaoHandler do + alias Cotacoes.Models.Cotacao + + @callback find_cotacoes_not_ingested :: list(Cotacao.t()) + @callback ingest_cotacoes(list(Cotacao.t())) :: :ok + @callback insert_cotacoes!(list(Cotacao.t())) :: {:ok, list(Cotacao.t()) | nil} + @callback reject_inserted_cotacoes(list(Cotacao.t())) :: list(Cotacao.t()) +end From 121ba6b9db2b6232729a58a773f150bf2eeb35a7 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:52:30 -0300 Subject: [PATCH 04/20] feat: adiciona campo em cotacao para verificar se a mesma ja foi importada --- apps/cotacoes/lib/cotacoes/models/cotacao.ex | 11 +++++++++-- apps/cotacoes/lib/cotacoes/repository.ex | 2 +- .../repo/migrations/20230704182917_cria_cotacao.exs | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/cotacoes/lib/cotacoes/models/cotacao.ex b/apps/cotacoes/lib/cotacoes/models/cotacao.ex index 9dd74843..7a61dc9f 100644 --- a/apps/cotacoes/lib/cotacoes/models/cotacao.ex +++ b/apps/cotacoes/lib/cotacoes/models/cotacao.ex @@ -1,16 +1,23 @@ defmodule Cotacoes.Models.Cotacao do use Database, :model - @type t :: %Cotacao{id: binary, data: Date.t(), link: binary, fonte: binary} + @type t :: %Cotacao{ + id: binary, + data: Date.t(), + link: binary, + fonte: binary, + importada?: boolean + } @required_fields ~w(data fonte)a - @optional_fields ~w(link)a + @optional_fields ~w(link ingested?)a @primary_key false schema "cotacao" do field :data, :date, primary_key: true field :fonte, :string, primary_key: true field :link, :string + field :importada?, :boolean, default: false field :id, Database.Types.PublicId, autogenerate: true end diff --git a/apps/cotacoes/lib/cotacoes/repository.ex b/apps/cotacoes/lib/cotacoes/repository.ex index 1be6d224..1368acb4 100644 --- a/apps/cotacoes/lib/cotacoes/repository.ex +++ b/apps/cotacoes/lib/cotacoes/repository.ex @@ -9,7 +9,7 @@ defmodule Cotacoes.Repository do @impl true def find_all_cotacao_by_is_ingested do - query = from c in Cotacao, where: c.ingested?, select: c + query = from c in Cotacao, where: c.importada?, select: c Repo.Replica.all(query) end diff --git a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs index 4099c733..3d5c21b3 100644 --- a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs +++ b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs @@ -6,6 +6,7 @@ defmodule Database.Repo.Migrations.CriaCotacao do add :id, :string add :data, :date, primary_key: true, null: false add :link, :string + add :importada?, :boolean, default: false add :fonte, references(:fonte_cotacao, column: :nome, type: :string), primary_key: true, From 296996f0a7996092a7aab121148f9ce85293182a Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:53:14 -0300 Subject: [PATCH 05/20] feat: remove app crawler --- apps/crawler/.formatter.exs | 4 ---- apps/crawler/.gitignore | 26 ------------------------ apps/crawler/README.md | 3 --- apps/crawler/mix.exs | 33 ------------------------------- apps/crawler/test/test_helper.exs | 1 - 5 files changed, 67 deletions(-) delete mode 100644 apps/crawler/.formatter.exs delete mode 100644 apps/crawler/.gitignore delete mode 100644 apps/crawler/README.md delete mode 100644 apps/crawler/mix.exs delete mode 100644 apps/crawler/test/test_helper.exs diff --git a/apps/crawler/.formatter.exs b/apps/crawler/.formatter.exs deleted file mode 100644 index d2cda26e..00000000 --- a/apps/crawler/.formatter.exs +++ /dev/null @@ -1,4 +0,0 @@ -# Used by "mix format" -[ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/apps/crawler/.gitignore b/apps/crawler/.gitignore deleted file mode 100644 index 98390be6..00000000 --- a/apps/crawler/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -# The directory Mix will write compiled artifacts to. -/_build/ - -# If you run "mix test --cover", coverage assets end up here. -/cover/ - -# The directory Mix downloads your dependencies sources to. -/deps/ - -# Where third-party dependencies like ExDoc output generated docs. -/doc/ - -# Ignore .fetch files in case you like to edit your project deps locally. -/.fetch - -# If the VM crashes, it generates a dump, let's ignore it too. -erl_crash.dump - -# Also ignore archive artifacts (built via "mix archive.build"). -*.ez - -# Ignore package tarball (built via "mix hex.build"). -crawler-*.tar - -# Temporary files, for example, from tests. -/tmp/ diff --git a/apps/crawler/README.md b/apps/crawler/README.md deleted file mode 100644 index e77b286d..00000000 --- a/apps/crawler/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Crawler - -**TODO: Add description** diff --git a/apps/crawler/mix.exs b/apps/crawler/mix.exs deleted file mode 100644 index 9bedf547..00000000 --- a/apps/crawler/mix.exs +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Crawler.MixProject do - use Mix.Project - - def project do - [ - app: :crawler, - version: "0.1.0", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.14", - start_permanent: Mix.env() == :prod, - deps: deps() - ] - end - - # Run "mix help compile.app" to learn about applications. - def application do - [ - extra_applications: [:logger] - ] - end - - # Run "mix help deps" to learn about dependencies. - defp deps do - [ - {:crawly, "~> 0.15.0"}, - {:floki, "~> 0.34.0"}, - {:explorer, "~> 0.5.0"} - ] - end -end diff --git a/apps/crawler/test/test_helper.exs b/apps/crawler/test/test_helper.exs deleted file mode 100644 index 869559e7..00000000 --- a/apps/crawler/test/test_helper.exs +++ /dev/null @@ -1 +0,0 @@ -ExUnit.start() From 28dcf69f1083b0fed80038488bea4603218156f0 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:54:34 -0300 Subject: [PATCH 06/20] feat: adiciona app cotacoes_etl --- apps/cotacoes_etl/.formatter.exs | 4 +++ apps/cotacoes_etl/.gitignore | 26 ++++++++++++++ apps/cotacoes_etl/README.md | 3 ++ .../schemas/pesagro/boletim_entry.ex | 25 ++++++++++++++ apps/cotacoes_etl/mix.exs | 34 +++++++++++++++++++ apps/cotacoes_etl/test/test_helper.exs | 1 + mix.exs | 1 + mix.lock | 7 ++++ 8 files changed, 101 insertions(+) create mode 100644 apps/cotacoes_etl/.formatter.exs create mode 100644 apps/cotacoes_etl/.gitignore create mode 100644 apps/cotacoes_etl/README.md create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/schemas/pesagro/boletim_entry.ex create mode 100644 apps/cotacoes_etl/mix.exs create mode 100644 apps/cotacoes_etl/test/test_helper.exs diff --git a/apps/cotacoes_etl/.formatter.exs b/apps/cotacoes_etl/.formatter.exs new file mode 100644 index 00000000..d2cda26e --- /dev/null +++ b/apps/cotacoes_etl/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/apps/cotacoes_etl/.gitignore b/apps/cotacoes_etl/.gitignore new file mode 100644 index 00000000..98390be6 --- /dev/null +++ b/apps/cotacoes_etl/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +crawler-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/apps/cotacoes_etl/README.md b/apps/cotacoes_etl/README.md new file mode 100644 index 00000000..3868a2dd --- /dev/null +++ b/apps/cotacoes_etl/README.md @@ -0,0 +1,3 @@ +# ETL Cotações + +**TODO: Add description** diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/pesagro/boletim_entry.ex b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/pesagro/boletim_entry.ex new file mode 100644 index 00000000..03881a4d --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/pesagro/boletim_entry.ex @@ -0,0 +1,25 @@ +defmodule CotacoesETL.Schemas.Pesagro.BoletimEntry do + use Ecto.Schema + import Ecto.Changeset + alias __MODULE__ + + @typep tipo :: :pdf | :zip + @type t :: %BoletimEntry{link: binary, arquivo: binary, tipo: tipo} + + @fields ~w(link arquivo tipo)a + + @primary_key false + embedded_schema do + field(:link, :string) + field(:arquivo, :string) + field(:tipo, Ecto.Enum, values: ~w(pdf zip)a) + end + + @spec changeset(map) :: BoletimEntry.t() + def changeset(attrs) do + %BoletimEntry{} + |> cast(attrs, @fields) + |> validate_required(@fields) + |> apply_action!(:parse) + end +end diff --git a/apps/cotacoes_etl/mix.exs b/apps/cotacoes_etl/mix.exs new file mode 100644 index 00000000..724644a1 --- /dev/null +++ b/apps/cotacoes_etl/mix.exs @@ -0,0 +1,34 @@ +defmodule CotacoesETL.MixProject do + use Mix.Project + + def project do + [ + app: :cotacoes_etl, + version: "0.1.0", + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + def application do + [ + extra_applications: [:logger] + ] + end + + defp deps do + [ + {:req, "~> 0.3.0"}, + {:ecto, "~> 3.10"}, + {:gen_stage, "~> 1.0"}, + {:floki, "~> 0.34.0"}, + {:explorer, "~> 0.5.0"}, + {:cotacoes, in_umbrella: true} + ] + end +end diff --git a/apps/cotacoes_etl/test/test_helper.exs b/apps/cotacoes_etl/test/test_helper.exs new file mode 100644 index 00000000..869559e7 --- /dev/null +++ b/apps/cotacoes_etl/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/mix.exs b/mix.exs index 28d061d0..64b8bff8 100644 --- a/mix.exs +++ b/mix.exs @@ -13,6 +13,7 @@ defmodule Pescarte.MixProject do applications: [ database: :permanent, cotacoes: :permanent, + cotacoes_etl: :permanent, proxy_web: :permanent, identidades: :permanent, modulo_pesquisa: :permanent, diff --git a/mix.lock b/mix.lock index 7e25a579..34203a35 100644 --- a/mix.lock +++ b/mix.lock @@ -32,7 +32,9 @@ "explorer": {:hex, :explorer, "0.5.7", "dd8d3dee6004ef18ca4cfaf9683f069b0dbc5fb84b758a3458b7dd8330ea88e5", [:mix], [{:nx, "~> 0.4.0 or ~> 0.5.0", [hex: :nx, repo: "hexpm", optional: true]}, {:rustler, "~> 0.28.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.5", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}, {:table, "~> 0.1.2", [hex: :table, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "b7e23182c72fc5d2c523ff55f54276719667c3257a00e25daea2f0e01bdea522"}, "expo": {:hex, :expo, "0.4.1", "1c61d18a5df197dfda38861673d392e642649a9cef7694d2f97a587b2cfb319b", [:mix], [], "hexpm", "2ff7ba7a798c8c543c12550fa0e2cbc81b95d4974c65855d8d15ba7b37a1ce47"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"}, "floki": {:hex, :floki, "0.34.3", "5e2dcaec5d7c228ce5b1d3501502e308b2d79eb655e4191751a1fe491c37feac", [:mix], [], "hexpm", "9577440eea5b97924b4bf3c7ea55f7b8b6dce589f9b28b096cc294a8dc342341"}, + "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.22.3", "c8273e78db4a0bb6fba7e9f0fd881112f349a3117f7f7c598fa18c66c888e524", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "935f23447713954a6866f1bb28c3a878c4c011e802bcd68a726f5e558e4b64bd"}, "gollum": {:hex, :new_gollum, "0.4.0", "89e3e2fc5abd032455341c4a03bcef7042b8d08e02c51df24b99a1a0a1ad69b1", [:mix], [{:httpoison, "~> 1.7", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "85c68465e8678637638656945677062a4e7086e91a04d5c4bca1027321c74582"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~> 2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, @@ -49,8 +51,12 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, + "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, "nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"}, + "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"}, + "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "phoenix": {:hex, :phoenix, "1.7.6", "61f0625af7c1d1923d582470446de29b008c0e07ae33d7a3859ede247ddaf59a", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "f6b4be7780402bb060cbc6e83f1b6d3f5673b674ba73cc4a7dd47db0322dfb88"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.2", "b21bd01fdeffcfe2fab49e4942aa938b6d3e89e93a480d4aee58085560a0bc0d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "70242edd4601d50b69273b057ecf7b684644c19ee750989fd555625ae4ce8f5d"}, @@ -67,6 +73,7 @@ "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "postgrex": {:hex, :postgrex, "0.17.1", "01c29fd1205940ee55f7addb8f1dc25618ca63a8817e56fac4f6846fc2cddcbe", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "14b057b488e73be2beee508fb1955d8db90d6485c6466428fe9ccf1d6692a555"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "req": {:hex, :req, "0.3.10", "71b6457cba0929deedb4ceb5747d46f6c0b4d6be1f94a91b2c6a90f3e4a892bd", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "43489aa0d5b899215088cd6fea4125569009304c24b15215e3d02883aa0e79d8"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.6.1", "160b545bce8bf9a3f1b436b2c10f53574036a0db628e40f393328cbbe593602f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "0dd269fa261c4e3df290b12031c575fff07a542749f7b0e8b744d72d66c43600"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "table": {:hex, :table, "0.1.2", "87ad1125f5b70c5dea0307aa633194083eb5182ec537efc94e96af08937e14a8", [:mix], [], "hexpm", "7e99bc7efef806315c7e65640724bf165c3061cdc5d854060f74468367065029"}, From 0b64d9945310e6ae57b130899282b0efa433157f Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:55:00 -0300 Subject: [PATCH 07/20] =?UTF-8?q?feat:=20cria=20integra=C3=A7=C3=A3o=20com?= =?UTF-8?q?=20site=20da=20pesagro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cotacoes_etl/integrations/pesagro_api.ex | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex new file mode 100644 index 00000000..009686cc --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex @@ -0,0 +1,35 @@ +defmodule CotacoesETL.Integrations.PesagroAPI do + alias CotacoesETL.Schemas.Pesagro.BoletimEntry + + @filetypes ~w(application/pdf application/zip) + + def base_url, do: "https://www.pesagro.rj.gov.br/node/194" + + @spec fetch_document! :: Floki.html_tree() + def fetch_document! do + raw_document = Req.get!(base_url()).body + + Floki.parse_document!(raw_document) + end + + @spec fetch_all_boletim_links(Floki.html_tree()) :: list(BoletimEntry.t()) + def fetch_all_boletim_links(document) do + document + |> Floki.find("a") + |> Enum.filter(&is_pdf_or_zip?/1) + |> Enum.map(&a_tag_to_boletim/1) + end + + defp a_tag_to_boletim(tag) do + arquivo = Floki.text(Floki.attribute(tag, "title")) + link = base_url() <> Floki.text(Floki.attribute(tag, "href")) + attrs = %{arquivo: arquivo, link: link} + + BoletimEntry.changeset(attrs) + end + + defp is_pdf_or_zip?(tag) do + type = tag |> Floki.attribute("type") |> Floki.text() + type in @filetypes + end +end From 958f3a4905b1b5a5f8bc9969618cf51a3890ebf7 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Thu, 6 Jul 2023 17:56:29 -0300 Subject: [PATCH 08/20] feat: adiciona worker para buscar boletins na pesagro --- .../cotacoes_etl/adapters/pesagro/boletim.ex | 17 +++++ .../workers/pesagro/boletins_fetcher.ex | 75 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex new file mode 100644 index 00000000..91376b17 --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex @@ -0,0 +1,17 @@ +defmodule CotacoesETL.Adapters.Pesagro.Boletim do + import Ecto.Changeset, only: [apply_action!: 2] + alias Cotacoes.Models.Cotacao + alias CotacoesETL.Schemas.Pesagro.BoletimEntry + + def boletins_to_cotacoes(boletins, today) do + Enum.map(boletins, &boletim_to_cotacao(&1, today)) + end + + def boletim_to_cotacao(%BoletimEntry{link: link}, today) do + attrs = %{fonte: "pesagro", link: link, data: today, ingested?: false} + + %Cotacao{} + |> Cotacao.changeset(attrs) + |> apply_action!(:adapt) + end +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex new file mode 100644 index 00000000..a6d0db6e --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -0,0 +1,75 @@ +defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do + @moduledoc """ + Todos os dias esse Worker visita a página da `Pesagro` e + busca novos boletins do Mercado Agrícola, a fim de atualizar + nossa base de dados. + """ + + use GenServer + + alias Cotacoes.Handlers.CotacaoHandler + alias CotacoesETL.Adapters.Pesagro.Boletim + alias CotacoesETL.Integrations.PesagroAPI + alias CotacoesETL.Workers.Pesagro.CotacaoIngester + + require Logger + + @one_day 864 * 100 * 100 * 10 + + def start_link do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(boletins) do + GenServer.cast(__MODULE__, :fetch) + {:ok, boletins} + end + + @impl true + def handle_cast(:fetch, boletins) do + Logger.info("[#{__MODULE__}] ==> Buscando boletins em Pesagro") + + document = PesagroAPI.fetch_document!() + boletins_links = PesagroAPI.fetch_all_boletim_links(document) + + # Envia mensagem para si próprio para inserir os boletins + GenServer.cast(__MODULE__, :insert) + + # Agenda a próxima execução do Fetcher + schedule_next_fetch() + {:noreply, boletins ++ boletins_links} + end + + def handle_cast(:insert, boletins) do + Logger.info("[#{__MODULE__}] ==> Inserindo novos boletins da Pesagro") + + today = Date.utc_today() + cotacoes = Boletim.boletins_to_cotacoes(boletins, today) + cotacoes_diff = CotacaoHandler.reject_inserted_cotacoes(cotacoes) + Logger.info("[#{__MODULE__}] ==> #{length(cotacoes_diff)} novos boletins achados em Pesagro") + + {:ok, inserted} = CotacaoHandler.insert_cotacoes!(cotacoes_diff) + Logger.info("[#{__MODULE__}] ==> #{length(inserted)} novas cotações inseridas") + + # Agenda a ingestão dos dados das cotações + # pelo worker `CotacaoIngester` + Logger.info("[#{__MODULE__}] ==> Agendando a ingestão das cotações Pesagro inseridas") + schedule_ingestion() + {:noreply, []} + end + + @impl true + def handle_info(:schedule_fetch, state) do + GenServer.cast(__MODULE__, :fetch) + {:noreply, state} + end + + defp schedule_ingestion do + Process.send(CotacaoIngester, :ingest, []) + end + + defp schedule_next_fetch do + Process.send_after(self(), :schedule_fetch, @one_day) + end +end From 0a27863255d387be1c933ef0c60abdb2d7d3c1bb Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Fri, 7 Jul 2023 00:16:48 -0300 Subject: [PATCH 09/20] testes basicos --- .../lib/cotacoes_etl/integrations.ex | 7 +++++ .../i_manage_pesagro_integration.ex | 3 ++ .../cotacoes_etl/integrations/pesagro_api.ex | 14 ++++++--- .../workers/pesagro/boletins_fetcher.ex | 10 +++---- apps/cotacoes_etl/mix.exs | 1 + .../adapters/pesagro/boletim_test.exs | 21 +++++++++++++ .../integrations/pesagro_api_test.exs | 30 +++++++++++++++++++ apps/cotacoes_etl/test/test_helper.exs | 4 +++ mix.lock | 1 + 9 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex create mode 100644 apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs create mode 100644 apps/cotacoes_etl/test/cotacoes_etl/integrations/pesagro_api_test.exs diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex new file mode 100644 index 00000000..275a26dc --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex @@ -0,0 +1,7 @@ +defmodule CotacoesETL.Integrations do + alias CotacoesETL.Integrations.PesagroAPI + + def pesagro_api do + Application.get_env(:cotacoes_etl, :pesagro_api, PesagroAPI) + end +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex new file mode 100644 index 00000000..986bd089 --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex @@ -0,0 +1,3 @@ +defmodule CotacoesETL.Integrations.IManagePesagroIntegration do + @callback fetch_document! :: Floki.html_tree() +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex index 009686cc..9b8ebb18 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex @@ -1,13 +1,17 @@ defmodule CotacoesETL.Integrations.PesagroAPI do + alias CotacoesETL.Integrations.IManagePesagroIntegration alias CotacoesETL.Schemas.Pesagro.BoletimEntry + @behaviour IManagePesagroIntegration + @filetypes ~w(application/pdf application/zip) + @path "/node/194" - def base_url, do: "https://www.pesagro.rj.gov.br/node/194" + def base_url, do: "https://www.pesagro.rj.gov.br" - @spec fetch_document! :: Floki.html_tree() + @impl true def fetch_document! do - raw_document = Req.get!(base_url()).body + raw_document = Req.get!(base_url() <> @path).body Floki.parse_document!(raw_document) end @@ -23,7 +27,9 @@ defmodule CotacoesETL.Integrations.PesagroAPI do defp a_tag_to_boletim(tag) do arquivo = Floki.text(Floki.attribute(tag, "title")) link = base_url() <> Floki.text(Floki.attribute(tag, "href")) - attrs = %{arquivo: arquivo, link: link} + type = tag |> Floki.attribute("type") |> Floki.text() + tipo = if type == "application/pdf", do: :pdf, else: :zip + attrs = %{arquivo: arquivo, link: link, tipo: tipo} BoletimEntry.changeset(attrs) end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex index a6d0db6e..e318d72f 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -9,7 +9,7 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do alias Cotacoes.Handlers.CotacaoHandler alias CotacoesETL.Adapters.Pesagro.Boletim - alias CotacoesETL.Integrations.PesagroAPI + alias CotacoesETL.Integrations alias CotacoesETL.Workers.Pesagro.CotacaoIngester require Logger @@ -30,8 +30,8 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do def handle_cast(:fetch, boletins) do Logger.info("[#{__MODULE__}] ==> Buscando boletins em Pesagro") - document = PesagroAPI.fetch_document!() - boletins_links = PesagroAPI.fetch_all_boletim_links(document) + document = Integrations.pesagro_api().fetch_document!() + boletins_links = Integrations.pesagro_api().fetch_all_boletim_links(document) # Envia mensagem para si próprio para inserir os boletins GenServer.cast(__MODULE__, :insert) @@ -48,9 +48,7 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do cotacoes = Boletim.boletins_to_cotacoes(boletins, today) cotacoes_diff = CotacaoHandler.reject_inserted_cotacoes(cotacoes) Logger.info("[#{__MODULE__}] ==> #{length(cotacoes_diff)} novos boletins achados em Pesagro") - - {:ok, inserted} = CotacaoHandler.insert_cotacoes!(cotacoes_diff) - Logger.info("[#{__MODULE__}] ==> #{length(inserted)} novas cotações inseridas") + :ok = CotacaoHandler.insert_cotacoes!(cotacoes_diff) # Agenda a ingestão dos dados das cotações # pelo worker `CotacaoIngester` diff --git a/apps/cotacoes_etl/mix.exs b/apps/cotacoes_etl/mix.exs index 724644a1..c65b8d0c 100644 --- a/apps/cotacoes_etl/mix.exs +++ b/apps/cotacoes_etl/mix.exs @@ -28,6 +28,7 @@ defmodule CotacoesETL.MixProject do {:gen_stage, "~> 1.0"}, {:floki, "~> 0.34.0"}, {:explorer, "~> 0.5.0"}, + {:mox, "~> 1.0", only: :test}, {:cotacoes, in_umbrella: true} ] end diff --git a/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs new file mode 100644 index 00000000..b2052b43 --- /dev/null +++ b/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs @@ -0,0 +1,21 @@ +defmodule CotacoesETL.Adapters.Pesagro.BoletimTest do + use ExUnit.Case, async: true + + alias Cotacoes.Models.Cotacao + alias CotacoesETL.Adapters.Pesagro.Boletim + alias CotacoesETL.Schemas.Pesagro.BoletimEntry + + @moduletag :unit + + test "dado um boletim entry, deve retornar uma Cotacao válida" do + link = "https://example.com/arquivo.pdf" + arquivo = "arquivo.pdf" + today = Date.utc_today() + boletim = %BoletimEntry{link: link, arquivo: arquivo} + + cotacao = Boletim.boletim_to_cotacao(boletim, today) + assert %Cotacao{fonte: "pesagro", importada?: false} = cotacao + assert cotacao.link == link + assert cotacao.data == today + end +end diff --git a/apps/cotacoes_etl/test/cotacoes_etl/integrations/pesagro_api_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/integrations/pesagro_api_test.exs new file mode 100644 index 00000000..283a3637 --- /dev/null +++ b/apps/cotacoes_etl/test/cotacoes_etl/integrations/pesagro_api_test.exs @@ -0,0 +1,30 @@ +defmodule CotacoesETL.Integrations.PesagroAPITest do + use ExUnit.Case, async: true + + import Mox + + alias CotacoesETL.Integrations + alias CotacoesETL.Integrations.PesagroAPI + alias CotacoesETL.Schemas.Pesagro.BoletimEntry + + @moduletag :integration + + setup :verify_on_exit! + + test "fetch_document!/0" do + expect(IManagePesagroIntegrationMock, :fetch_document!, fn -> + {"html", [], + [ + {"a", [{"href", "/home"}, {"type", "application/zip"}, {"title", "arquivo.zip"}], + ["hello"]} + ]} + end) + + assert document = Integrations.pesagro_api().fetch_document!() + [boletim] = PesagroAPI.fetch_all_boletim_links(document) + + assert %BoletimEntry{} = boletim + assert boletim.link == "https://www.pesagro.rj.gov.br/home" + assert boletim.arquivo == "arquivo.zip" + end +end diff --git a/apps/cotacoes_etl/test/test_helper.exs b/apps/cotacoes_etl/test/test_helper.exs index 869559e7..e0df9da3 100644 --- a/apps/cotacoes_etl/test/test_helper.exs +++ b/apps/cotacoes_etl/test/test_helper.exs @@ -1 +1,5 @@ +alias CotacoesETL.Integrations.IManagePesagroIntegration +Mox.defmock(IManagePesagroIntegrationMock, for: IManagePesagroIntegration) +Application.put_env(:cotacoes_etl, :pesagro_api, IManagePesagroIntegrationMock) + ExUnit.start() diff --git a/mix.lock b/mix.lock index 34203a35..e41d1b10 100644 --- a/mix.lock +++ b/mix.lock @@ -52,6 +52,7 @@ "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, + "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, "nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"}, "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, From d9bedc48f66a887206205844979226df26582198 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Fri, 7 Jul 2023 01:03:47 -0300 Subject: [PATCH 10/20] feat: teste do fluxo principal do worker --- .../lib/cotacoes/handlers/cotacao_handler.ex | 22 ++------- .../handlers/i_manage_cotacao_handler.ex | 3 +- .../handlers/cotacao_handler_test.exs | 22 +++++++++ .../cotacoes_etl/adapters/pesagro/boletim.ex | 8 +--- .../workers/pesagro/boletins_fetcher.ex | 23 +++++++--- .../adapters/pesagro/boletim_test.exs | 3 +- .../workers/pesagro/boletins_fetcher_test.exs | 46 +++++++++++++++++++ 7 files changed, 93 insertions(+), 34 deletions(-) create mode 100644 apps/cotacoes/test/cotacoes/handlers/cotacao_handler_test.exs create mode 100644 apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs diff --git a/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex index eb48d9f1..a1e2173f 100644 --- a/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex +++ b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex @@ -3,6 +3,9 @@ defmodule Cotacoes.Handlers.CotacaoHandler do @behaviour Cotacoes.Handlers.IManageCotacaoHandler + @impl true + defdelegate list_cotacao, to: Repository + @impl true def find_cotacoes_not_ingested do Repository.find_all_cotacao_by_is_ingested() @@ -21,22 +24,7 @@ defmodule Cotacoes.Handlers.CotacaoHandler do @impl true def reject_inserted_cotacoes(cotacoes) do - cotacoes_ids = - cotacoes - |> Enum.sort_by(& &1.id_publico) - |> Enum.map(& &1.link) - |> MapSet.new() - - current = Repository.list_cotacao() - - current_ids = - current - |> Enum.sort_by(& &1.id_publico) - |> Enum.map(& &1.link) - |> MapSet.new() - - diff_ids = MapSet.difference(current_ids, cotacoes_ids) - - Enum.filter(cotacoes ++ current, &(&1 in diff_ids)) + current = Enum.map(Repository.list_cotacao(), & &1.link) + Enum.reject(cotacoes, &(&1.link in current)) end end diff --git a/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex index 3c477229..0c672b4c 100644 --- a/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex +++ b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex @@ -3,6 +3,7 @@ defmodule Cotacoes.Handlers.IManageCotacaoHandler do @callback find_cotacoes_not_ingested :: list(Cotacao.t()) @callback ingest_cotacoes(list(Cotacao.t())) :: :ok - @callback insert_cotacoes!(list(Cotacao.t())) :: {:ok, list(Cotacao.t()) | nil} + @callback insert_cotacoes!(list(Cotacao.t())) :: :ok + @callback list_cotacao :: list(Cotacao.t()) @callback reject_inserted_cotacoes(list(Cotacao.t())) :: list(Cotacao.t()) end diff --git a/apps/cotacoes/test/cotacoes/handlers/cotacao_handler_test.exs b/apps/cotacoes/test/cotacoes/handlers/cotacao_handler_test.exs new file mode 100644 index 00000000..551ba66f --- /dev/null +++ b/apps/cotacoes/test/cotacoes/handlers/cotacao_handler_test.exs @@ -0,0 +1,22 @@ +defmodule Cotacoes.Handlers.CotacaoHandlerTest do + use Database.DataCase, async: true + + import Cotacoes.Factory + + alias Cotacoes.Handlers.CotacaoHandler + + @moduletag :unit + + describe "reject_inserted_cotacoes/1" do + test "quando não existem cotacoes" do + cotacao = fixture(:cotacao) + assert [^cotacao] = CotacaoHandler.reject_inserted_cotacoes([cotacao]) + end + + test "quando existem cotacoes" do + insert(:cotacao) + cotacao = fixture(:cotacao) + assert [^cotacao] = CotacaoHandler.reject_inserted_cotacoes([cotacao]) + end + end +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex index 91376b17..47b1b1c1 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex @@ -1,6 +1,4 @@ defmodule CotacoesETL.Adapters.Pesagro.Boletim do - import Ecto.Changeset, only: [apply_action!: 2] - alias Cotacoes.Models.Cotacao alias CotacoesETL.Schemas.Pesagro.BoletimEntry def boletins_to_cotacoes(boletins, today) do @@ -8,10 +6,6 @@ defmodule CotacoesETL.Adapters.Pesagro.Boletim do end def boletim_to_cotacao(%BoletimEntry{link: link}, today) do - attrs = %{fonte: "pesagro", link: link, data: today, ingested?: false} - - %Cotacao{} - |> Cotacao.changeset(attrs) - |> apply_action!(:adapt) + %{fonte: "pesagro", link: link, data: today, importada?: false} end end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex index e318d72f..be9d0490 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -10,7 +10,7 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do alias Cotacoes.Handlers.CotacaoHandler alias CotacoesETL.Adapters.Pesagro.Boletim alias CotacoesETL.Integrations - alias CotacoesETL.Workers.Pesagro.CotacaoIngester + alias CotacoesETL.Integrations.PesagroAPI require Logger @@ -20,18 +20,27 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do GenServer.start_link(__MODULE__, [], name: __MODULE__) end + def get_current_boletins do + GenServer.call(__MODULE__, :get_current) + end + @impl true def init(boletins) do GenServer.cast(__MODULE__, :fetch) {:ok, boletins} end + @impl true + def handle_call(:get_current, _from, state) do + {:reply, state, state} + end + @impl true def handle_cast(:fetch, boletins) do Logger.info("[#{__MODULE__}] ==> Buscando boletins em Pesagro") document = Integrations.pesagro_api().fetch_document!() - boletins_links = Integrations.pesagro_api().fetch_all_boletim_links(document) + boletins_links = PesagroAPI.fetch_all_boletim_links(document) # Envia mensagem para si próprio para inserir os boletins GenServer.cast(__MODULE__, :insert) @@ -52,8 +61,8 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do # Agenda a ingestão dos dados das cotações # pelo worker `CotacaoIngester` - Logger.info("[#{__MODULE__}] ==> Agendando a ingestão das cotações Pesagro inseridas") - schedule_ingestion() + # Logger.info("[#{__MODULE__}] ==> Agendando a ingestão das cotações Pesagro inseridas") + # schedule_ingestion() {:noreply, []} end @@ -63,9 +72,9 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do {:noreply, state} end - defp schedule_ingestion do - Process.send(CotacaoIngester, :ingest, []) - end + # defp schedule_ingestion do + # Process.send(CotacaoIngester, :ingest, []) + # end defp schedule_next_fetch do Process.send_after(self(), :schedule_fetch, @one_day) diff --git a/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs index b2052b43..20488097 100644 --- a/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs +++ b/apps/cotacoes_etl/test/cotacoes_etl/adapters/pesagro/boletim_test.exs @@ -1,7 +1,6 @@ defmodule CotacoesETL.Adapters.Pesagro.BoletimTest do use ExUnit.Case, async: true - alias Cotacoes.Models.Cotacao alias CotacoesETL.Adapters.Pesagro.Boletim alias CotacoesETL.Schemas.Pesagro.BoletimEntry @@ -14,7 +13,7 @@ defmodule CotacoesETL.Adapters.Pesagro.BoletimTest do boletim = %BoletimEntry{link: link, arquivo: arquivo} cotacao = Boletim.boletim_to_cotacao(boletim, today) - assert %Cotacao{fonte: "pesagro", importada?: false} = cotacao + assert %{fonte: "pesagro", importada?: false} = cotacao assert cotacao.link == link assert cotacao.data == today end diff --git a/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs new file mode 100644 index 00000000..31ae4878 --- /dev/null +++ b/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs @@ -0,0 +1,46 @@ +defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcherTest do + use ExUnit.Case, async: false + + import Mox + import Cotacoes.Factory + + alias Cotacoes.Handlers.CotacaoHandler + alias CotacoesETL.Schemas.Pesagro.BoletimEntry + alias CotacoesETL.Workers.Pesagro.BoletinsFetcher + + @moduletag :integration + + setup :set_mox_from_context + setup :verify_on_exit! + + setup tags do + Database.DataCase.setup_sandbox(tags) + :ok + end + + test "fluxo buscador de boletins pesagro" do + insert(:fonte, nome: "pesagro") + + expect(IManagePesagroIntegrationMock, :fetch_document!, fn -> + {"html", [], + [ + {"a", [{"href", "/home"}, {"type", "application/zip"}, {"title", "arquivo.zip"}], + ["hello"]} + ]} + end) + + assert {:ok, _pid} = BoletinsFetcher.start_link() + [boletim] = BoletinsFetcher.get_current_boletins() + + assert %BoletimEntry{} = boletim + assert boletim.link == "https://www.pesagro.rj.gov.br/home" + assert boletim.arquivo == "arquivo.zip" + + assert :sys.get_state(BoletinsFetcher) + + assert [cotacao] = CotacaoHandler.list_cotacao() + refute cotacao.importada? + assert cotacao.fonte == "pesagro" + assert cotacao.link == "https://www.pesagro.rj.gov.br/home" + end +end From 7e82f7cdb86ecb36997d01cda382213205eb498e Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 00:51:26 -0300 Subject: [PATCH 11/20] corrige estrutura da tabela cotacao --- apps/cotacoes/lib/cotacoes/models/cotacao.ex | 7 +++---- apps/cotacoes/lib/cotacoes/models/cotacao_pescado.ex | 8 ++++---- .../repo/migrations/20230704182917_cria_cotacao.exs | 11 ++++------- .../20230704182918_cria_cotacoes_pescados.exs | 6 +++--- .../test/cotacoes/models/cotacao_pescado_test.exs | 10 +++++----- apps/cotacoes/test/support/factory.ex | 2 +- 6 files changed, 20 insertions(+), 24 deletions(-) diff --git a/apps/cotacoes/lib/cotacoes/models/cotacao.ex b/apps/cotacoes/lib/cotacoes/models/cotacao.ex index 9ea49348..535b9245 100644 --- a/apps/cotacoes/lib/cotacoes/models/cotacao.ex +++ b/apps/cotacoes/lib/cotacoes/models/cotacao.ex @@ -12,11 +12,10 @@ defmodule Cotacoes.Models.Cotacao do @required_fields ~w(data fonte)a @optional_fields ~w(link importada?)a - @primary_key false + @primary_key {:link, :string, autogenerate: false} schema "cotacao" do - field :data, :date, primary_key: true - field :fonte, :string, primary_key: true - field :link, :string + field :data, :date + field :fonte, :string field :importada?, :boolean, default: false field :id, Database.Types.PublicId, autogenerate: true end diff --git a/apps/cotacoes/lib/cotacoes/models/cotacao_pescado.ex b/apps/cotacoes/lib/cotacoes/models/cotacao_pescado.ex index 8b20f070..20537b1f 100644 --- a/apps/cotacoes/lib/cotacoes/models/cotacao_pescado.ex +++ b/apps/cotacoes/lib/cotacoes/models/cotacao_pescado.ex @@ -3,7 +3,7 @@ defmodule Cotacoes.Models.CotacaoPescado do @type t :: %CotacaoPescado{ id: binary, - cotacao_data: Date.t(), + cotacao_link: Date.t(), pescado_codigo: binary, fonte_nome: binary, preco_minimo: integer, @@ -12,13 +12,13 @@ defmodule Cotacoes.Models.CotacaoPescado do preco_medio: integer } - @required_fields ~w(cotacao_data pescado_codigo fonte_nome preco_minimo preco_maximo)a + @required_fields ~w(cotacao_link pescado_codigo fonte_nome preco_minimo preco_maximo)a @optional_fields ~w(preco_mais_comum preco_medio)a @primary_key false schema "cotacoes_pescados" do field :id, Database.Types.PublicId, autogenerate: true - field :cotacao_data, :date, primary_key: true + field :cotacao_link, :string, primary_key: true field :pescado_codigo, :string, primary_key: true field :fonte_nome, :string, primary_key: true field :preco_minimo, :integer @@ -32,7 +32,7 @@ defmodule Cotacoes.Models.CotacaoPescado do cot_pescado |> cast(attrs, @required_fields ++ @optional_fields) |> validate_required(@required_fields) - |> foreign_key_constraint(:cotacao_data) + |> foreign_key_constraint(:cotacao_link) |> foreign_key_constraint(:pescado_codigo) |> foreign_key_constraint(:fonte_nome) end diff --git a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs index 3d5c21b3..9b0658e2 100644 --- a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs +++ b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs @@ -4,16 +4,13 @@ defmodule Database.Repo.Migrations.CriaCotacao do def change do create table(:cotacao, primary_key: false) do add :id, :string - add :data, :date, primary_key: true, null: false - add :link, :string + add :data, :date, null: false + add :link, :string, primary_key: true add :importada?, :boolean, default: false - - add :fonte, references(:fonte_cotacao, column: :nome, type: :string), - primary_key: true, - null: false + add :fonte, references(:fonte_cotacao, column: :nome, type: :string), null: false end create index(:cotacao, [:fonte]) - create unique_index(:cotacao, [:data]) + create index(:cotacao, [:link]) end end diff --git a/apps/cotacoes/priv/repo/migrations/20230704182918_cria_cotacoes_pescados.exs b/apps/cotacoes/priv/repo/migrations/20230704182918_cria_cotacoes_pescados.exs index 90106ab5..e14995bd 100644 --- a/apps/cotacoes/priv/repo/migrations/20230704182918_cria_cotacoes_pescados.exs +++ b/apps/cotacoes/priv/repo/migrations/20230704182918_cria_cotacoes_pescados.exs @@ -4,7 +4,7 @@ defmodule Database.Repo.Migrations.CriaCotacoesPescados do def change do create table(:cotacoes_pescados, primary_key: false) do add :id, :string - add :cotacao_data, references(:cotacao, column: :data, type: :date), primary_key: true + add :cotacao_link, references(:cotacao, column: :link, type: :string), primary_key: true add :pescado_codigo, references(:pescado, column: :codigo, type: :string), primary_key: true add :fonte_nome, references(:fonte_cotacao, column: :nome, type: :string), primary_key: true add :preco_minimo, :integer @@ -13,9 +13,9 @@ defmodule Database.Repo.Migrations.CriaCotacoesPescados do add :preco_medio, :integer end - create index(:cotacoes_pescados, [:cotacao_data]) + create index(:cotacoes_pescados, [:cotacao_link]) create index(:cotacoes_pescados, [:pescado_codigo]) create index(:cotacoes_pescados, [:fonte_nome]) - create unique_index(:cotacoes_pescados, [:cotacao_data, :pescado_codigo, :fonte_nome]) + create unique_index(:cotacoes_pescados, [:cotacao_link, :pescado_codigo, :fonte_nome]) end end diff --git a/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs b/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs index 28b8d0f2..7c1a52e3 100644 --- a/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs +++ b/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs @@ -13,7 +13,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do pescado = insert(:pescado) attrs = %{ - cotacao_data: cotacao.data, + cotacao_link: cotacao.data, fonte_nome: fonte.nome, pescado_codigo: pescado.codigo, preco_minimo: 1000, @@ -23,7 +23,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do changeset = CotacaoPescado.changeset(%CotacaoPescado{}, attrs) assert changeset.valid? - assert get_change(changeset, :cotacao_data) == cotacao.data + assert get_change(changeset, :cotacao_link) == cotacao.data assert get_change(changeset, :fonte_nome) == fonte.nome assert get_change(changeset, :pescado_codigo) == pescado.codigo assert get_change(changeset, :preco_minimo) == 1000 @@ -36,7 +36,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do pescado = insert(:pescado) attrs = %{ - cotacao_data: cotacao.data, + cotacao_link: cotacao.data, fonte_nome: fonte.nome, pescado_codigo: pescado.codigo, preco_minimo: 1000, @@ -48,7 +48,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do changeset = CotacaoPescado.changeset(%CotacaoPescado{}, attrs) assert changeset.valid? - assert get_change(changeset, :cotacao_data) == cotacao.data + assert get_change(changeset, :cotacao_link) == cotacao.data assert get_change(changeset, :fonte_nome) == fonte.nome assert get_change(changeset, :pescado_codigo) == pescado.codigo assert get_change(changeset, :preco_minimo) == 1000 @@ -61,7 +61,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do changeset = CotacaoPescado.changeset(%CotacaoPescado{}, %{}) refute changeset.valid? - assert Keyword.get(changeset.errors, :cotacao_data) + assert Keyword.get(changeset.errors, :cotacao_link) assert Keyword.get(changeset.errors, :fonte_nome) assert Keyword.get(changeset.errors, :pescado_codigo) assert Keyword.get(changeset.errors, :preco_maximo) diff --git a/apps/cotacoes/test/support/factory.ex b/apps/cotacoes/test/support/factory.ex index b3748a03..31be5579 100644 --- a/apps/cotacoes/test/support/factory.ex +++ b/apps/cotacoes/test/support/factory.ex @@ -19,7 +19,7 @@ defmodule Cotacoes.Factory do def cotacao_pescado_factory do %CotacaoPescado{ id: Nanoid.generate_non_secure(), - cotacao_data: insert(:cotacao).data, + cotacao_link: insert(:cotacao).link, fonte_nome: insert(:fonte).nome, pescado_codigo: insert(:pescado).codigo, preco_minimo: 1000, From 2a036bf1a30ce5b3ed63d310336108c828144b3b Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 00:52:04 -0300 Subject: [PATCH 12/20] remove lib req e adiciona tesla --- .../lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex | 5 +++-- apps/cotacoes_etl/mix.exs | 5 ++++- config/config.exs | 2 ++ mix.lock | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex index be9d0490..0bed1a55 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -15,8 +15,9 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do require Logger @one_day 864 * 100 * 100 * 10 + @half_minute 30 * 100 * 10 - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end @@ -26,7 +27,7 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do @impl true def init(boletins) do - GenServer.cast(__MODULE__, :fetch) + Process.send_after(__MODULE__, :schedule_fetch, @half_minute) {:ok, boletins} end diff --git a/apps/cotacoes_etl/mix.exs b/apps/cotacoes_etl/mix.exs index c65b8d0c..42fa3397 100644 --- a/apps/cotacoes_etl/mix.exs +++ b/apps/cotacoes_etl/mix.exs @@ -17,13 +17,16 @@ defmodule CotacoesETL.MixProject do def application do [ + mod: {CotacoesETL.Application, []}, extra_applications: [:logger] ] end defp deps do [ - {:req, "~> 0.3.0"}, + {:tesla, "~> 1.4"}, + {:finch, "~> 0.16"}, + {:jason, ">= 1.0.0"}, {:ecto, "~> 3.10"}, {:gen_stage, "~> 1.0"}, {:floki, "~> 0.34.0"}, diff --git a/config/config.exs b/config/config.exs index b364db14..0887b3bd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -2,6 +2,8 @@ import Config config :database, config_env: config_env() +config :tesla, adapter: {Tesla.Adapter.Finch, name: PescarteHTTPClient} + # -------- # # Database # # -------- # diff --git a/mix.lock b/mix.lock index e41d1b10..560c7fe2 100644 --- a/mix.lock +++ b/mix.lock @@ -83,6 +83,7 @@ "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"}, + "tesla": {:hex, :tesla, "1.7.0", "a62dda2f80d4f8a925eb7b8c5b78c461e0eb996672719fe1a63b26321a5f8b4e", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "2e64f01ebfdb026209b47bc651a0e65203fcff4ae79c11efb73c4852b00dc313"}, "thousand_island": {:hex, :thousand_island, "0.6.7", "3a91a7e362ca407036c6691e8a4f6e01ac8e901db3598875863a149279ac8571", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "541a5cb26b88adf8d8180b6b96a90f09566b4aad7a6b3608dcac969648cf6765"}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, From 9672045a55da905ef21ce2f03f30fd87bb4ea867 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 00:52:44 -0300 Subject: [PATCH 13/20] adiciona schemas para API zamzar --- .../lib/cotacoes_etl/schemas/zamzar/file.ex | 34 ++++++++++++++ .../lib/cotacoes_etl/schemas/zamzar/job.ex | 44 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex new file mode 100644 index 00000000..a4f74421 --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex @@ -0,0 +1,34 @@ +defmodule CotacoesETL.Schemas.Zamzar.File do + use Ecto.Schema + import Ecto.Changeset + alias __MODULE__ + + @type t :: %File{ + id: integer, + key: String.t(), + size: integer, + name: String.t(), + path: Path.t(), + format: String.t(), + created_at: NaiveDateTime.t() + } + + @fields ~w(id key size name format created_at)a + + @primary_key false + embedded_schema do + field(:id, :integer) + field(:key, :string) + field(:size, :integer) + field(:name, :string) + field(:path, :string) + field(:format, :string) + field(:created_at, :naive_datetime) + end + + def changeset(file \\ %File{}, attrs) do + file + |> cast(attrs, @fields) + |> apply_action!(:parse) + end +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex new file mode 100644 index 00000000..c9dca703 --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex @@ -0,0 +1,44 @@ +defmodule CotacoesETL.Schemas.Zamzar.Job do + use Ecto.Schema + import Ecto.Changeset + alias CotacoesETL.Schemas.Zamzar.File + alias __MODULE__ + + @type t :: %Job{ + id: integer, + sandbox: boolean, + key: String.t(), + created_at: NaiveDateTime.t(), + finished_at: NaiveDateTime.t(), + target_format: String.t(), + status: atom, + source_file: File.t(), + target_files: list(File.t()) + } + + @fields ~w(id sandbox key created_at finished_at target_format credit_cost status)a + @status ~w(initialising successful failed)a + + @primary_key false + embedded_schema do + field(:id, :integer) + field(:sandbox, :boolean) + field(:key, :string) + field(:created_at, :naive_datetime) + field(:finished_at, :naive_datetime) + field(:target_format, :string) + field(:credit_cost, :integer) + field(:status, Ecto.Enum, values: @status) + + embeds_one(:source_file, File) + embeds_many(:target_files, File) + end + + def changeset(attrs) do + %Job{} + |> cast(attrs, @fields) + |> cast_embed(:source_file, with: &File.changeset/2, required: true) + |> cast_embed(:target_files, with: &File.changeset/2, required: false) + |> apply_action!(:parse) + end +end From 247a26d8e2035cf953e4630ff5c7cbb55d4b2c88 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 00:54:05 -0300 Subject: [PATCH 14/20] =?UTF-8?q?adiciona=20integra=C3=A7=C3=A3o=20com=20a?= =?UTF-8?q?=20API=20zamzar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/cotacoes_etl/application.ex | 12 ++++ .../i_manage_zamzar_integration.ex | 18 ++++++ .../cotacoes_etl/integrations/pesagro_api.ex | 2 +- .../cotacoes_etl/integrations/zamzar_api.ex | 64 +++++++++++++++++++ config/runtime.exs | 2 + 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/application.ex create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex create mode 100644 apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/application.ex b/apps/cotacoes_etl/lib/cotacoes_etl/application.ex new file mode 100644 index 00000000..ecf760dc --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/application.ex @@ -0,0 +1,12 @@ +defmodule CotacoesETL.Application do + use Application + + alias CotacoesETL.Workers.Pesagro.BoletinsFetcher + + @impl true + def start(_, _) do + children = [BoletinsFetcher, {Finch, name: PescarteHTTPClient}] + opts = [strategy: :one_for_one] + Supervisor.start_link(children, opts) + end +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex new file mode 100644 index 00000000..2b7065e4 --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex @@ -0,0 +1,18 @@ +defmodule CotacoesETL.Integrations.IManageZamzarIntegration do + alias CotacoesETL.Schemas.Zamzar.Job + alias CotacoesETL.Schemas.Zamzar.File, as: FileEntry + + @callback start_job!(source_path, target_format) :: Job.t() + when source_path: Path.t(), + target_format: String.t() + + @callback retrieve_job!(job_id) :: Job.t() + when job_id: integer + + @callback download_converted_file!(file_id, target_path) :: binary + when file_id: integer, + target_path: Path.t() + + @callback retrieve_file_info!(file_id) :: FileEntry.t() + when file_id: integer +end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex index 9b8ebb18..42855cc9 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex @@ -11,7 +11,7 @@ defmodule CotacoesETL.Integrations.PesagroAPI do @impl true def fetch_document! do - raw_document = Req.get!(base_url() <> @path).body + raw_document = Tesla.get!(base_url() <> @path).body Floki.parse_document!(raw_document) end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex new file mode 100644 index 00000000..f1de596d --- /dev/null +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex @@ -0,0 +1,64 @@ +defmodule CotacoesETL.Integrations.ZamzarAPI do + use Tesla + + alias CotacoesETL.Integrations.IManageZamzarIntegration + alias CotacoesETL.Schemas.Zamzar.File, as: FileEntry + alias CotacoesETL.Schemas.Zamzar.Job + alias Tesla.Multipart + + @behaviour IManageZamzarIntegration + + plug(Tesla.Middleware.BaseUrl, "https://api.zamzar.com/v1") + plug(Tesla.Middleware.BasicAuth, username: zamzar_api_key(), password: "") + # Ninguém pode me deter! + plug(Tesla.Middleware.FormUrlencoded, encode: &URI.encode_query/1, decode: &Jason.decode/1) + plug(Tesla.Middleware.FollowRedirects, max_redirects: 2) + + @impl true + def start_job!(source_path, target_format) do + multipart = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_field("target_format", target_format) + |> Multipart.add_file(source_path) + + "/jobs" + |> post!(multipart) + |> Map.fetch!(:body) + |> Job.changeset() + end + + @impl true + def retrieve_job!(job_id) do + "/jobs/#{job_id}" + |> get!() + |> Map.fetch!(:body) + |> Job.changeset() + end + + @impl true + def download_converted_file!(file_id, target_path) do + file_content = + "/files/#{file_id}/content" + |> get!() + |> Map.fetch!(:body) + + File.write!(target_path, file_content) + + metadata = retrieve_file_info!(file_id) + + FileEntry.changeset(metadata, %{path: target_path}) + end + + @impl true + def retrieve_file_info!(file_id) do + "/files/#{file_id}" + |> get!() + |> Map.fetch!(:body) + |> FileEntry.changeset() + end + + defp zamzar_api_key do + Application.get_env(:cotacoes_etl, :zamzar_api_key) + end +end diff --git a/config/runtime.exs b/config/runtime.exs index b9aeffbc..a05c1947 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -6,6 +6,8 @@ if System.get_env("PHX_SERVER") do config :proxy_web, ProxyWeb.Endpoint, server: true end +config :cotacoes_etl, zamzar_api_key: System.fetch_env!("ZAMZAR_API_KEY") + if config_env() == :prod do database_url = System.get_env("DATABASE_URL") || From 88503d67be4b32774c9588573bb60f58f146de8c Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 01:13:02 -0300 Subject: [PATCH 15/20] =?UTF-8?q?pequenos=20ajustes=20na=20integra=C3=A7?= =?UTF-8?q?=C3=A3o=20da=20API=20zamzar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integrations/i_manage_zamzar_integration.ex | 4 ++-- .../lib/cotacoes_etl/integrations/zamzar_api.ex | 15 ++++++++++----- .../lib/cotacoes_etl/schemas/zamzar/file.ex | 8 ++++++-- config/runtime.exs | 6 +++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex index 2b7065e4..d289bcd8 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_zamzar_integration.ex @@ -1,6 +1,6 @@ defmodule CotacoesETL.Integrations.IManageZamzarIntegration do - alias CotacoesETL.Schemas.Zamzar.Job alias CotacoesETL.Schemas.Zamzar.File, as: FileEntry + alias CotacoesETL.Schemas.Zamzar.Job @callback start_job!(source_path, target_format) :: Job.t() when source_path: Path.t(), @@ -9,7 +9,7 @@ defmodule CotacoesETL.Integrations.IManageZamzarIntegration do @callback retrieve_job!(job_id) :: Job.t() when job_id: integer - @callback download_converted_file!(file_id, target_path) :: binary + @callback download_converted_file!(file_id, target_path) :: FileEntry.t() when file_id: integer, target_path: Path.t() diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex index f1de596d..904c87b6 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/zamzar_api.ex @@ -8,10 +8,8 @@ defmodule CotacoesETL.Integrations.ZamzarAPI do @behaviour IManageZamzarIntegration - plug(Tesla.Middleware.BaseUrl, "https://api.zamzar.com/v1") + plug(Tesla.Middleware.BaseUrl, zamzar_endpoint()) plug(Tesla.Middleware.BasicAuth, username: zamzar_api_key(), password: "") - # Ninguém pode me deter! - plug(Tesla.Middleware.FormUrlencoded, encode: &URI.encode_query/1, decode: &Jason.decode/1) plug(Tesla.Middleware.FollowRedirects, max_redirects: 2) @impl true @@ -20,11 +18,12 @@ defmodule CotacoesETL.Integrations.ZamzarAPI do Multipart.new() |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_field("target_format", target_format) - |> Multipart.add_file(source_path) + |> Multipart.add_file(source_path, name: "source_file") "/jobs" |> post!(multipart) |> Map.fetch!(:body) + |> Jason.decode!() |> Job.changeset() end @@ -33,6 +32,7 @@ defmodule CotacoesETL.Integrations.ZamzarAPI do "/jobs/#{job_id}" |> get!() |> Map.fetch!(:body) + |> Jason.decode!() |> Job.changeset() end @@ -47,7 +47,7 @@ defmodule CotacoesETL.Integrations.ZamzarAPI do metadata = retrieve_file_info!(file_id) - FileEntry.changeset(metadata, %{path: target_path}) + FileEntry.changeset!(metadata, %{path: target_path}) end @impl true @@ -55,9 +55,14 @@ defmodule CotacoesETL.Integrations.ZamzarAPI do "/files/#{file_id}" |> get!() |> Map.fetch!(:body) + |> Jason.decode!() |> FileEntry.changeset() end + defp zamzar_endpoint do + Application.get_env(:cotacoes_etl, :zamzar_endpoint) + end + defp zamzar_api_key do Application.get_env(:cotacoes_etl, :zamzar_api_key) end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex index a4f74421..edb5ef7f 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/file.ex @@ -13,7 +13,7 @@ defmodule CotacoesETL.Schemas.Zamzar.File do created_at: NaiveDateTime.t() } - @fields ~w(id key size name format created_at)a + @fields ~w(id key size name format created_at path)a @primary_key false embedded_schema do @@ -27,8 +27,12 @@ defmodule CotacoesETL.Schemas.Zamzar.File do end def changeset(file \\ %File{}, attrs) do + cast(file, attrs, @fields) + end + + def changeset!(file \\ %File{}, attrs) do file - |> cast(attrs, @fields) + |> changeset(attrs) |> apply_action!(:parse) end end diff --git a/config/runtime.exs b/config/runtime.exs index a05c1947..d2db56c2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -6,9 +6,13 @@ if System.get_env("PHX_SERVER") do config :proxy_web, ProxyWeb.Endpoint, server: true end -config :cotacoes_etl, zamzar_api_key: System.fetch_env!("ZAMZAR_API_KEY") +config :cotacoes_etl, + zamzar_api_key: System.fetch_env!("ZAMZAR_API_KEY"), + zamzar_endpoint: System.get_env("ZAMZAR_ENDPOINT", "https://sandbox.zamzar.com/v1") if config_env() == :prod do + config :cotacoes_etl, zamzar_endpoint: System.fetch_env!("ZAMZAR_ENDPOINT") + database_url = System.get_env("DATABASE_URL") || raise """ From b92052fb01884fa7e601e79d2c352cfb22ff8aa9 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 12:26:17 -0300 Subject: [PATCH 16/20] =?UTF-8?q?add=20baixada=3F=20ao=20modelo=20de=20cot?= =?UTF-8?q?acao=20e=20implementa=20fun=C3=A7=C3=B5es=20extras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/cotacoes/handlers/cotacao_handler.ex | 19 ++++++++++++++++++- .../handlers/i_manage_cotacao_handler.ex | 4 ++++ .../lib/cotacoes/i_manage_repository.ex | 3 ++- apps/cotacoes/lib/cotacoes/models/cotacao.ex | 6 ++++-- apps/cotacoes/lib/cotacoes/repository.ex | 10 ++++++++-- .../20230704182917_cria_cotacao.exs | 1 + apps/cotacoes/test/support/factory.ex | 3 ++- 7 files changed, 39 insertions(+), 7 deletions(-) diff --git a/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex index a1e2173f..06626d3d 100644 --- a/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex +++ b/apps/cotacoes/lib/cotacoes/handlers/cotacao_handler.ex @@ -8,7 +8,19 @@ defmodule Cotacoes.Handlers.CotacaoHandler do @impl true def find_cotacoes_not_ingested do - Repository.find_all_cotacao_by_is_ingested() + Repository.find_all_cotacao_by_not_ingested() + end + + @impl true + def find_cotacoes_not_downloaded do + Repository.find_all_cotacao_by_not_downloaded() + end + + @impl true + def get_cotacao_file_base_name(cotacao) do + cotacao.link + |> String.split("/") + |> List.last() end @impl true @@ -27,4 +39,9 @@ defmodule Cotacoes.Handlers.CotacaoHandler do current = Enum.map(Repository.list_cotacao(), & &1.link) Enum.reject(cotacoes, &(&1.link in current)) end + + @impl true + def set_cotacao_downloaded(cotacao) do + Repository.upsert_cotacao(cotacao, %{baixada?: true}) + end end diff --git a/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex index 0c672b4c..718d5d2d 100644 --- a/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex +++ b/apps/cotacoes/lib/cotacoes/handlers/i_manage_cotacao_handler.ex @@ -2,8 +2,12 @@ defmodule Cotacoes.Handlers.IManageCotacaoHandler do alias Cotacoes.Models.Cotacao @callback find_cotacoes_not_ingested :: list(Cotacao.t()) + @callback find_cotacoes_not_downloaded :: list(Cotacao.t()) + @callback get_cotacao_file_base_name(Cotacao.t()) :: String.t() @callback ingest_cotacoes(list(Cotacao.t())) :: :ok @callback insert_cotacoes!(list(Cotacao.t())) :: :ok @callback list_cotacao :: list(Cotacao.t()) @callback reject_inserted_cotacoes(list(Cotacao.t())) :: list(Cotacao.t()) + @callback set_cotacao_downloaded(Cotacao.t()) :: + {:ok, Cotacao.t()} | {:error, Ecto.Changeset.t()} end diff --git a/apps/cotacoes/lib/cotacoes/i_manage_repository.ex b/apps/cotacoes/lib/cotacoes/i_manage_repository.ex index 80a8d78c..c74aeeb8 100644 --- a/apps/cotacoes/lib/cotacoes/i_manage_repository.ex +++ b/apps/cotacoes/lib/cotacoes/i_manage_repository.ex @@ -5,7 +5,8 @@ defmodule Cotacoes.IManageRepository do @opaque changeset :: Ecto.Changeset.t() - @callback find_all_cotacao_by_is_ingested :: list(Cotacao.t()) + @callback find_all_cotacao_by_not_ingested :: list(Cotacao.t()) + @callback find_all_cotacao_by_not_downloaded :: list(Cotacao.t()) @callback insert_all_cotacao(list(map)) :: :ok @callback list_cotacao :: list(Cotacao.t()) @callback update_all_cotacao(list(Cotacao.t()), keyword) :: {:ok, list(Cotacao.t()) | nil} diff --git a/apps/cotacoes/lib/cotacoes/models/cotacao.ex b/apps/cotacoes/lib/cotacoes/models/cotacao.ex index 535b9245..ea297e64 100644 --- a/apps/cotacoes/lib/cotacoes/models/cotacao.ex +++ b/apps/cotacoes/lib/cotacoes/models/cotacao.ex @@ -6,17 +6,19 @@ defmodule Cotacoes.Models.Cotacao do data: Date.t(), link: binary, fonte: binary, - importada?: boolean + importada?: boolean, + baixada?: boolean } @required_fields ~w(data fonte)a - @optional_fields ~w(link importada?)a + @optional_fields ~w(link importada? baixada?)a @primary_key {:link, :string, autogenerate: false} schema "cotacao" do field :data, :date field :fonte, :string field :importada?, :boolean, default: false + field :baixada?, :boolean, default: false field :id, Database.Types.PublicId, autogenerate: true end diff --git a/apps/cotacoes/lib/cotacoes/repository.ex b/apps/cotacoes/lib/cotacoes/repository.ex index 63072f5a..883b7fc4 100644 --- a/apps/cotacoes/lib/cotacoes/repository.ex +++ b/apps/cotacoes/lib/cotacoes/repository.ex @@ -8,8 +8,14 @@ defmodule Cotacoes.Repository do @behaviour Cotacoes.IManageRepository @impl true - def find_all_cotacao_by_is_ingested do - query = from c in Cotacao, where: c.importada?, select: c + def find_all_cotacao_by_not_ingested do + query = from c in Cotacao, where: not c.importada?, select: c + Repo.Replica.all(query) + end + + @impl true + def find_all_cotacao_by_not_downloaded do + query = from c in Cotacao, where: not c.baixada?, select: c Repo.Replica.all(query) end diff --git a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs index 9b0658e2..be6dac2a 100644 --- a/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs +++ b/apps/cotacoes/priv/repo/migrations/20230704182917_cria_cotacao.exs @@ -7,6 +7,7 @@ defmodule Database.Repo.Migrations.CriaCotacao do add :data, :date, null: false add :link, :string, primary_key: true add :importada?, :boolean, default: false + add :baixada?, :boolean, default: false add :fonte, references(:fonte_cotacao, column: :nome, type: :string), null: false end diff --git a/apps/cotacoes/test/support/factory.ex b/apps/cotacoes/test/support/factory.ex index 31be5579..284ff653 100644 --- a/apps/cotacoes/test/support/factory.ex +++ b/apps/cotacoes/test/support/factory.ex @@ -12,7 +12,8 @@ defmodule Cotacoes.Factory do data: ~D[2023-05-07], fonte: insert(:fonte).nome, link: sequence(:link, &"https://example#{&1}.com"), - importada?: false + importada?: false, + baixada?: false } end From 4622af2c65df5bd88990f5c03fb53dc69d69e6c1 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 12:30:52 -0300 Subject: [PATCH 17/20] =?UTF-8?q?fun=C3=A7=C3=B5es=20extras=20na=20integra?= =?UTF-8?q?=C3=A7=C3=A3o=20pesagro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/cotacoes_etl/adapters/pesagro/boletim.ex | 2 +- apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex | 5 +++++ .../integrations/i_manage_pesagro_integration.ex | 2 ++ .../lib/cotacoes_etl/integrations/pesagro_api.ex | 5 +++++ apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex | 2 +- .../lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex | 6 +++--- apps/cotacoes_etl/mix.exs | 1 + mix.lock | 2 ++ 8 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex index 47b1b1c1..0848e950 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/adapters/pesagro/boletim.ex @@ -6,6 +6,6 @@ defmodule CotacoesETL.Adapters.Pesagro.Boletim do end def boletim_to_cotacao(%BoletimEntry{link: link}, today) do - %{fonte: "pesagro", link: link, data: today, importada?: false} + %{id: Nanoid.generate(), fonte: "pesagro", link: link, data: today, importada?: false} end end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex index 275a26dc..4de3f779 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations.ex @@ -1,7 +1,12 @@ defmodule CotacoesETL.Integrations do alias CotacoesETL.Integrations.PesagroAPI + alias CotacoesETL.Integrations.ZamzarAPI def pesagro_api do Application.get_env(:cotacoes_etl, :pesagro_api, PesagroAPI) end + + def zamzar_api do + Application.get_env(:cotacoes_etl, :zamzar_api, ZamzarAPI) + end end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex index 986bd089..b87bad4d 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/i_manage_pesagro_integration.ex @@ -1,3 +1,5 @@ defmodule CotacoesETL.Integrations.IManagePesagroIntegration do @callback fetch_document! :: Floki.html_tree() + @callback download_file!(link) :: binary + when link: String.t() end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex index 42855cc9..a1a55642 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/integrations/pesagro_api.ex @@ -16,6 +16,11 @@ defmodule CotacoesETL.Integrations.PesagroAPI do Floki.parse_document!(raw_document) end + @impl true + def download_file!(link) do + Tesla.get!(link).body + end + @spec fetch_all_boletim_links(Floki.html_tree()) :: list(BoletimEntry.t()) def fetch_all_boletim_links(document) do document diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex index c9dca703..887e0b60 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/schemas/zamzar/job.ex @@ -17,7 +17,7 @@ defmodule CotacoesETL.Schemas.Zamzar.Job do } @fields ~w(id sandbox key created_at finished_at target_format credit_cost status)a - @status ~w(initialising successful failed)a + @status ~w(initialising successful failed converting)a @primary_key false embedded_schema do diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex index 0bed1a55..ae477b04 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -73,9 +73,9 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do {:noreply, state} end - # defp schedule_ingestion do - # Process.send(CotacaoIngester, :ingest, []) - # end + defp schedule_ingestion do + Process.send(CotacaoIngester, :schedule_ingestion, []) + end defp schedule_next_fetch do Process.send_after(self(), :schedule_fetch, @one_day) diff --git a/apps/cotacoes_etl/mix.exs b/apps/cotacoes_etl/mix.exs index 42fa3397..4f16caa5 100644 --- a/apps/cotacoes_etl/mix.exs +++ b/apps/cotacoes_etl/mix.exs @@ -31,6 +31,7 @@ defmodule CotacoesETL.MixProject do {:gen_stage, "~> 1.0"}, {:floki, "~> 0.34.0"}, {:explorer, "~> 0.5.0"}, + {:unzip, "~> 0.8"}, {:mox, "~> 1.0", only: :test}, {:cotacoes, in_umbrella: true} ] diff --git a/mix.lock b/mix.lock index 560c7fe2..0aa7ab96 100644 --- a/mix.lock +++ b/mix.lock @@ -88,8 +88,10 @@ "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "unzip": {:hex, :unzip, "0.8.0", "ee21d87c21b01567317387dab4228ac570ca15b41cfc221a067354cbf8e68c4d", [:mix], [], "hexpm", "ffa67a483efcedcb5876971a50947222e104d5f8fea2c4a0441e6f7967854827"}, "websock": {:hex, :websock, "0.5.2", "b3c08511d8d79ed2c2f589ff430bd1fe799bb389686dafce86d28801783d8351", [:mix], [], "hexpm", "925f5de22fca6813dfa980fb62fd542ec43a2d1a1f83d2caec907483fe66ff05"}, "websock_adapter": {:hex, :websock_adapter, "0.5.3", "4908718e42e4a548fc20e00e70848620a92f11f7a6add8cf0886c4232267498d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "cbe5b814c1f86b6ea002b52dd99f345aeecf1a1a6964e209d208fb404d930d3d"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, + "zstream": {:hex, :zstream, "0.6.4", "169ce887a443d4163085ee682ab1b0ad38db8fa45e843927b9b431a92f4b7d9e", [:mix], [], "hexpm", "acc6c35b6db9eb2cfe8b85e972cb9dc1b730f8efeb76c5bbe871216fe639d9a1"}, } From 14390d9d5cfa101abb3d93166835354b7f3f1d17 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 12:56:51 -0300 Subject: [PATCH 18/20] testes para API zamzar --- .../test/cotacoes/repository_test.exs | 10 +-- .../integrations/zamzar_api_test.exs | 68 +++++++++++++++++++ apps/cotacoes_etl/test/test_helper.exs | 4 ++ 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 apps/cotacoes_etl/test/cotacoes_etl/integrations/zamzar_api_test.exs diff --git a/apps/cotacoes/test/cotacoes/repository_test.exs b/apps/cotacoes/test/cotacoes/repository_test.exs index 513b6bf6..f650e3c3 100644 --- a/apps/cotacoes/test/cotacoes/repository_test.exs +++ b/apps/cotacoes/test/cotacoes/repository_test.exs @@ -8,13 +8,13 @@ defmodule Cotacoes.RepositoryTest do @moduletag :unit - test "find_all_cotacao_by_is_ingested/0 retorna todas as cotacoes onde importada? é true" do - insert(:cotacao, importada?: true) + test "find_all_cotacao_by_not_ingested/0 retorna todas as cotacoes onde importada? é false" do + insert(:cotacao, importada?: false) insert(:cotacao, importada?: true, data: ~D[2023-06-07]) - found = Repository.find_all_cotacao_by_is_ingested() + found = Repository.find_all_cotacao_by_not_ingested() - assert length(found) == 2 - assert Enum.all?(found, & &1.importada?) + assert length(found) == 1 + assert Enum.all?(found, &(not &1.importada?)) end test "insert_all_cotacao/1 insere todas as cotacoes fornecidas" do diff --git a/apps/cotacoes_etl/test/cotacoes_etl/integrations/zamzar_api_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/integrations/zamzar_api_test.exs new file mode 100644 index 00000000..58c7b15d --- /dev/null +++ b/apps/cotacoes_etl/test/cotacoes_etl/integrations/zamzar_api_test.exs @@ -0,0 +1,68 @@ +defmodule CotacoesETL.Integrations.ZamzarAPITest do + use ExUnit.Case, async: true + + alias CotacoesETL.Schemas.Zamzar.File, as: FileEntry + alias CotacoesETL.Schemas.Zamzar.Job + + import CotacoesETL.Integrations, only: [zamzar_api: 0] + import Mox + + setup :verify_on_exit! + + @moduletag :integration + + @job_mock %Job{ + id: 15, + key: "d62630b3b08643d2a76acaa34f453f60c8439252", + status: :initialising, + sandbox: true, + created_at: ~N[2013-10-27T13:41:00Z], + finished_at: nil, + source_file: %FileEntry{id: 2, name: "portrait.gif", size: 90_571}, + target_files: [], + target_format: "png", + credit_cost: 1 + } + + describe "start_job!/2" do + test "quando recebe um caminho de arquivo existente e um formato, deve retorna um Job" do + expect(IManageZamzarIntegrationMock, :start_job!, fn source, format -> + assert is_binary(source) + assert is_binary(format) + + @job_mock + end) + + assert %Job{} = zamzar_api().start_job!("/tmp/portrait.gif", "pdf") + end + end + + describe "retrieve_job!/1" do + expect(IManageZamzarIntegrationMock, :retrieve_job!, fn job_id -> + assert is_integer(job_id) + + %{@job_mock | target_files: [%FileEntry{id: 3, name: "portrait.pdf"}]} + end) + + assert %Job{} = job = zamzar_api().retrieve_job!(@job_mock.id) + assert job.id == @job_mock.id + refute Enum.empty?(job.target_files) + end + + describe "download_converted_file!/2" do + expect(IManageZamzarIntegrationMock, :download_converted_file!, fn file_id, target_path -> + assert is_integer(file_id) + File.write!(target_path, "teste") + + %FileEntry{id: 4, name: "portrait.pdf", path: target_path} + end) + + file = %FileEntry{id: 4, name: "portrait.gif"} + assert %FileEntry{} = file = zamzar_api().download_converted_file!(file.id, "/tmp/teste.pdf") + assert file.path == "/tmp/teste.pdf" + assert file.name == "portrait.pdf" + assert File.exists?("/tmp/teste.pdf") + + File.rm!("/tmp/teste.pdf") + end +end diff --git a/apps/cotacoes_etl/test/test_helper.exs b/apps/cotacoes_etl/test/test_helper.exs index e0df9da3..ab6f4886 100644 --- a/apps/cotacoes_etl/test/test_helper.exs +++ b/apps/cotacoes_etl/test/test_helper.exs @@ -2,4 +2,8 @@ alias CotacoesETL.Integrations.IManagePesagroIntegration Mox.defmock(IManagePesagroIntegrationMock, for: IManagePesagroIntegration) Application.put_env(:cotacoes_etl, :pesagro_api, IManagePesagroIntegrationMock) +alias CotacoesETL.Integrations.IManageZamzarIntegration +Mox.defmock(IManageZamzarIntegrationMock, for: IManageZamzarIntegration) +Application.put_env(:cotacoes_etl, :zamzar_api, IManageZamzarIntegrationMock) + ExUnit.start() From f49db91765e0af744d17e52ada7bf21faaca8260 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 13:04:15 -0300 Subject: [PATCH 19/20] fix: corrige testes --- .../test/cotacoes/models/cotacao_pescado_test.exs | 8 ++++---- apps/cotacoes_etl/lib/cotacoes_etl/application.ex | 12 +++++++++++- .../cotacoes_etl/workers/pesagro/boletins_fetcher.ex | 4 ++++ .../workers/pesagro/boletins_fetcher_test.exs | 3 ++- config/config.exs | 1 + 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs b/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs index 7c1a52e3..80e74637 100644 --- a/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs +++ b/apps/cotacoes/test/cotacoes/models/cotacao_pescado_test.exs @@ -13,7 +13,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do pescado = insert(:pescado) attrs = %{ - cotacao_link: cotacao.data, + cotacao_link: cotacao.link, fonte_nome: fonte.nome, pescado_codigo: pescado.codigo, preco_minimo: 1000, @@ -23,7 +23,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do changeset = CotacaoPescado.changeset(%CotacaoPescado{}, attrs) assert changeset.valid? - assert get_change(changeset, :cotacao_link) == cotacao.data + assert get_change(changeset, :cotacao_link) == cotacao.link assert get_change(changeset, :fonte_nome) == fonte.nome assert get_change(changeset, :pescado_codigo) == pescado.codigo assert get_change(changeset, :preco_minimo) == 1000 @@ -36,7 +36,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do pescado = insert(:pescado) attrs = %{ - cotacao_link: cotacao.data, + cotacao_link: cotacao.link, fonte_nome: fonte.nome, pescado_codigo: pescado.codigo, preco_minimo: 1000, @@ -48,7 +48,7 @@ defmodule Cotacoes.Models.CotacaoPescadoTest do changeset = CotacaoPescado.changeset(%CotacaoPescado{}, attrs) assert changeset.valid? - assert get_change(changeset, :cotacao_link) == cotacao.data + assert get_change(changeset, :cotacao_link) == cotacao.link assert get_change(changeset, :fonte_nome) == fonte.nome assert get_change(changeset, :pescado_codigo) == pescado.codigo assert get_change(changeset, :preco_minimo) == 1000 diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/application.ex b/apps/cotacoes_etl/lib/cotacoes_etl/application.ex index ecf760dc..93953e81 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/application.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/application.ex @@ -5,8 +5,18 @@ defmodule CotacoesETL.Application do @impl true def start(_, _) do - children = [BoletinsFetcher, {Finch, name: PescarteHTTPClient}] + children = + if config_env() != :test do + [BoletinsFetcher, {Finch, name: PescarteHTTPClient}] + else + [{Finch, name: PescarteHTTPClient}] + end + opts = [strategy: :one_for_one] Supervisor.start_link(children, opts) end + + defp config_env do + Application.get_env(:cotacoes_etl, :config_env) + end end diff --git a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex index ae477b04..9b297661 100644 --- a/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex +++ b/apps/cotacoes_etl/lib/cotacoes_etl/workers/pesagro/boletins_fetcher.ex @@ -25,6 +25,10 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcher do GenServer.call(__MODULE__, :get_current) end + def trigger_fetching do + GenServer.cast(__MODULE__, :fetch) + end + @impl true def init(boletins) do Process.send_after(__MODULE__, :schedule_fetch, @half_minute) diff --git a/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs b/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs index 31ae4878..6e667ca2 100644 --- a/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs +++ b/apps/cotacoes_etl/test/cotacoes_etl/workers/pesagro/boletins_fetcher_test.exs @@ -29,7 +29,8 @@ defmodule CotacoesETL.Workers.Pesagro.BoletinsFetcherTest do ]} end) - assert {:ok, _pid} = BoletinsFetcher.start_link() + assert {:ok, _pid} = BoletinsFetcher.start_link([]) + assert :ok = BoletinsFetcher.trigger_fetching() [boletim] = BoletinsFetcher.get_current_boletins() assert %BoletimEntry{} = boletim diff --git a/config/config.exs b/config/config.exs index 0887b3bd..eee9c656 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,6 +1,7 @@ import Config config :database, config_env: config_env() +config :cotacoes_etl, config_env: config_env() config :tesla, adapter: {Tesla.Adapter.Finch, name: PescarteHTTPClient} From b72668965139d5971708f728312301520e11c758 Mon Sep 17 00:00:00 2001 From: Zoey de Souza Pessanha Date: Sat, 8 Jul 2023 13:16:10 -0300 Subject: [PATCH 20/20] adiciona valor default em ambiente de teste pra API_KEY Zamzar --- config/runtime.exs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index d2db56c2..b7c33c2d 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -6,8 +6,16 @@ if System.get_env("PHX_SERVER") do config :proxy_web, ProxyWeb.Endpoint, server: true end +default_value_for_test_env = fn env_var -> + if config_env() == :test do + System.put_env(env_var, "não faz diferença") + else + System.fetch_env!(env_var) + end +end + config :cotacoes_etl, - zamzar_api_key: System.fetch_env!("ZAMZAR_API_KEY"), + zamzar_api_key: default_value_for_test_env.("ZAMZAR_API_KEY"), zamzar_endpoint: System.get_env("ZAMZAR_ENDPOINT", "https://sandbox.zamzar.com/v1") if config_env() == :prod do