From 9003c5cedf05411af184fa242305c11fd2620051 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Mon, 8 Jul 2024 11:06:10 +0300 Subject: [PATCH 1/4] &all/0 for all classes --- lib/location/city.ex | 5 +++++ lib/location/subdivision.ex | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/lib/location/city.ex b/lib/location/city.ex index 917afcf..4a5f275 100644 --- a/lib/location/city.ex +++ b/lib/location/city.ex @@ -48,6 +48,11 @@ defmodule Location.City do |> Stream.run() end + def all() do + :ets.tab2list(@ets_table_by_id) + |> Enum.map(fn {id, {name, country_code}} -> to_struct(id, name, country_code) end) + end + @doc """ Finds city by GeoNames ID. """ diff --git a/lib/location/subdivision.ex b/lib/location/subdivision.ex index a7803b1..ce4ace8 100644 --- a/lib/location/subdivision.ex +++ b/lib/location/subdivision.ex @@ -30,6 +30,11 @@ defmodule Location.Subdivision do end) end + def all() do + :ets.tab2list(@ets_table) + |> Enum.map(fn {_, entry} -> entry end) + end + def search_subdivision(search_phrase) do search_phrase = String.downcase(search_phrase) From 184cc16df90ca62f7a7e1d8df1fe428864925c55 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 9 Jul 2024 15:00:17 +0300 Subject: [PATCH 2/4] Save version of the library This is useful for users to know when the data has changed --- lib/location.ex | 6 ++++++ mix_tasks/scraper.ex | 7 +++++++ mix_tasks/update_english_translations.ex | 2 ++ mix_tasks/update_geoname_data.ex | 2 ++ mix_tasks/update_iso_data.ex | 2 ++ priv/version | 1 + 6 files changed, 20 insertions(+) create mode 100644 mix_tasks/scraper.ex create mode 100644 priv/version diff --git a/lib/location.ex b/lib/location.ex index 70acebf..01370f6 100644 --- a/lib/location.ex +++ b/lib/location.ex @@ -28,4 +28,10 @@ defmodule Location do Logger.debug("Loading location database #{inspect(module)} took: #{time}s") :ok end + + def version() do + version_file = Application.app_dir(:location, "priv/version") + + File.read!(version_file) + end end diff --git a/mix_tasks/scraper.ex b/mix_tasks/scraper.ex new file mode 100644 index 0000000..e488b1d --- /dev/null +++ b/mix_tasks/scraper.ex @@ -0,0 +1,7 @@ +defmodule Location.Scraper do + @version_file Application.app_dir(:location, "/priv/version") + + def write_date_to_version() do + File.write!(@version_file, Date.to_iso8601(Date.utc_today())) + end +end diff --git a/mix_tasks/update_english_translations.ex b/mix_tasks/update_english_translations.ex index 2a9c007..5f717b5 100644 --- a/mix_tasks/update_english_translations.ex +++ b/mix_tasks/update_english_translations.ex @@ -88,6 +88,8 @@ defmodule Mix.Tasks.UpdateEnglishTranslations do |> Jason.encode_to_iodata!(pretty: true) File.write!(@translations_dest, json) + + Location.Scraper.write_date_to_version() end manual_code_to_en_name = %{ diff --git a/mix_tasks/update_geoname_data.ex b/mix_tasks/update_geoname_data.ex index 712c8e4..49e2f34 100644 --- a/mix_tasks/update_geoname_data.ex +++ b/mix_tasks/update_geoname_data.ex @@ -30,6 +30,8 @@ defmodule Mix.Tasks.UpdateGeonameData do IO.puts("Writing result to #{@allcountries_dest}") File.write!(@allcountries_dest, Enum.join(result, "\n")) + + Location.Scraper.write_date_to_version() end defp reduce_chunk(row, result) do diff --git a/mix_tasks/update_iso_data.ex b/mix_tasks/update_iso_data.ex index a5d7353..a09a710 100644 --- a/mix_tasks/update_iso_data.ex +++ b/mix_tasks/update_iso_data.ex @@ -50,5 +50,7 @@ defmodule Mix.Tasks.UpdateIsoData do new_subdivisions = Jason.encode_to_iodata!(%{"3166-2" => new_subdivisions}, pretty: true) File.write!(@subdivisions_dest, new_subdivisions) + + Location.Scraper.write_date_to_version() end end diff --git a/priv/version b/priv/version new file mode 100644 index 0000000..f69ed48 --- /dev/null +++ b/priv/version @@ -0,0 +1 @@ +2024-07-09 \ No newline at end of file From d1fb29b42d89e9565c42dd769ab53391dac35fef Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 10 Jul 2024 11:00:21 +0300 Subject: [PATCH 3/4] Comment --- mix_tasks/update_geoname_data.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix_tasks/update_geoname_data.ex b/mix_tasks/update_geoname_data.ex index 49e2f34..51a4159 100644 --- a/mix_tasks/update_geoname_data.ex +++ b/mix_tasks/update_geoname_data.ex @@ -1,7 +1,7 @@ defmodule Mix.Tasks.UpdateGeonameData do use Mix.Task - @allcountries_src "https://download.geonames.org/export/dump/allCountries.zip" + # @allcountries_src "https://download.geonames.org/export/dump/allCountries.zip" @allcountries_dest Application.app_dir(:location, "/priv/geonames.csv") @doc """ From 7f23ab5935d385c428f970de85c74ae0730b8004 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 11 Jul 2024 16:12:16 +0300 Subject: [PATCH 4/4] Add tests to &all/0 --- test/location_test.exs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/location_test.exs b/test/location_test.exs index fe2c304..edbb4d7 100644 --- a/test/location_test.exs +++ b/test/location_test.exs @@ -21,6 +21,21 @@ defmodule LocationTest do assert match.alpha_2 == "EE" assert match.name == "Estonia" end + + test "&all/0" do + countries = Location.Country.all() + + assert length(countries) > 200 and length(countries) < 1_000 + + country = Enum.find(countries, &(&1.alpha_2 == "EE")) + + assert country == %Location.Country{ + alpha_2: "EE", + alpha_3: "EST", + name: "Estonia", + flag: "🇪🇪" + } + end end describe "subdivision" do @@ -146,6 +161,21 @@ defmodule LocationTest do # LU assert Location.get_subdivision("LU-CL").name == "Clervaux" end + + test "&all/0" do + subdivisions = Location.Subdivision.all() + + assert length(subdivisions) > 1_000 and length(subdivisions) < 10_000 + + subdivision = Enum.find(subdivisions, &(&1.code == "EE-79")) + + assert subdivision == %Location.Subdivision{ + name: "Tartumaa", + code: "EE-79", + country_code: "EE", + type: "County" + } + end end describe "city" do @@ -176,5 +206,14 @@ defmodule LocationTest do assert city.id == 8_349_432 assert city.name == "Springfield" end + + test "&all/0" do + cities = Location.City.all() + + assert length(cities) > 100_000 and length(cities) < 1_000_000 + + city = Enum.find(cities, &(&1.id == 588_335)) + assert city == %Location.City{country_code: "EE", name: "Tartu", id: 588_335} + end end end