add postal code

README.md

@@ -15,3 +15,10 @@ The data for subdivisions comes primarily from the [debian iso-codes](https://sa
 
 The data for cities comes from the [geonames](http://www.geonames.org/) project. This project has scripts to download the main `allCountries.txt` file. It is then processed to make it smaller
 (from 1.3GB to about 130MB). Still, the resulting file is quite large so we also provide a city database based on the smaller `cities500.txt` file.
+
+### Postal Codes
+
+The data for postal codes comes from the [geonames](http://www.geonames.org/) project. This project has scripts to download all or individual postal code files via the --source option.
+
+### Mix Tasks
+one must first refresh the applications task lists by running `mix compile` tasks can then be viewed via `mix help`

lib/location.ex

@@ -1,3 +1,4 @@
+NimbleCSV.define(PostCodeCSV, separator: ",", escape: "\~")
 NimbleCSV.define(LocationCSV, separator: "\t", escape: "\~")
 
 defmodule Location do
@@ -8,16 +9,27 @@ defmodule Location do
   defdelegate search_subdivision(code), to: Location.Subdivision
   defdelegate get_city(code), to: Location.City
   defdelegate get_city(city_name, country_code), to: Location.City
+  defdelegate get_postal_code(code), to: Location.PostalCode
+  defdelegate get_postal_codes(country_code, state_code, city_name), to: Location.PostalCode
+  defdelegate get_postal_codes(), to: Location.PostalCode
+
+  def unload_all()do
+    :ok = unload(Location.Country)
+    :ok = unload(Location.Subdivision)
+    :ok = unload(Location.City)
+    :ok = unload(Location.PostalCode)
+  end
 
   def load_all() do
     Logger.debug("Loading location databases...")
 
     :ok = load(Location.Country)
     :ok = load(Location.Subdivision)
     :ok = load(Location.City)
+    :ok = load(Location.PostalCode)
   end
 
-  defp load(module) do
+  def load(module) do
     {t, _result} =
       :timer.tc(fn ->
         module.load()
@@ -29,6 +41,13 @@ defmodule Location do
     :ok
   end
 
+  def unload(module) do
+    module.unload()
+
+    Logger.debug("Unloading location database #{inspect(module)}")
+    :ok
+  end
+
   def version() do
     version_file = Application.app_dir(:location, "priv/version")
 

lib/location/city.ex

@@ -4,6 +4,11 @@ defmodule Location.City do
 
   defstruct [:id, :name, :country_code]
 
+  def unload()do
+    :ets.delete(@ets_table_by_id)
+    :ets.delete(@ets_table_by_label)
+  end
+
   def load() do
     @ets_table_by_id =
       :ets.new(@ets_table_by_id, [

lib/location/country.ex

@@ -3,6 +3,10 @@ defmodule Location.Country do
 
   defstruct [:alpha_2, :alpha_3, :name, :flag]
 
+  def unload()do
+    :ets.delete(@ets_table)
+  end
+
   def load() do
     ets = :ets.new(@ets_table, [:named_table])
 

lib/location/postalcode.ex

@@ -0,0 +1,236 @@
+defmodule Location.PostalCode do
+  @ets_table_by_id __MODULE__
+  @ets_table_by_lookup Module.concat(__MODULE__, ByLookup)
+
+  defstruct [
+    :postal_code,
+    :country_code,
+    :state_code,
+    :city_name,
+    :latitude,
+    :longitude
+  ]
+
+  def unload()do
+    :ets.delete(@ets_table_by_id)
+    :ets.delete(@ets_table_by_lookup)
+  end
+
+  def load() do
+    @ets_table_by_lookup =
+      :ets.new(@ets_table_by_lookup, [
+        :set,
+        :named_table,
+        :public,
+        :compressed,
+        {:write_concurrency, true},
+        {:read_concurrency, true},
+        {:decentralized_counters, false}
+      ])
+
+    @ets_table_by_id =
+      :ets.new(@ets_table_by_id, [
+        :set,
+        :named_table,
+        :public,
+        :compressed,
+        {:write_concurrency, true},
+        {:read_concurrency, true},
+        {:decentralized_counters, false}
+      ])
+
+    source_file()
+    |> File.stream!()
+    |> Stream.chunk_every(15_000)
+    |> Task.async_stream(
+      fn chunk ->
+        chunk
+        |> PostCodeCSV.parse_stream()
+        |> Stream.each(fn data ->
+          __MODULE__.parse(data)
+        end)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
+    |> Stream.run()
+  end
+
+  def parse(data) do
+    case data do
+      [
+        country_code,
+        postal_code,
+        city_name,
+        _state_name,
+        state_code,
+        _municipality,
+        _municipality_code,
+        _admin_name3,
+        _admin_code3,
+        latitude,
+        longitude,
+        _accuracy,
+        _,
+        _
+      ] ->
+        country_code = String.trim(country_code)
+
+        true =
+          :ets.insert(
+            @ets_table_by_lookup,
+            {{country_code, state_code, city_name}, {postal_code, latitude, longitude}}
+          )
+
+        true =
+          :ets.insert(
+            @ets_table_by_id,
+            {postal_code, {country_code, state_code, city_name, latitude, longitude}}
+          )
+
+      [
+        country_code,
+        postal_code,
+        city_name,
+        _state_name,
+        state_code,
+        _municipality,
+        _municipality_code,
+        _admin_name3,
+        _admin_code3,
+        latitude,
+        longitude,
+        _accuracy,
+        _
+      ] ->
+        country_code = String.trim(country_code)
+
+        true =
+          :ets.insert(
+            @ets_table_by_lookup,
+            {{country_code, state_code, city_name}, {postal_code, latitude, longitude}}
+          )
+
+        true =
+          :ets.insert(
+            @ets_table_by_id,
+            {postal_code, {country_code, state_code, city_name, latitude, longitude}}
+          )
+
+      [
+        country_code,
+        postal_code,
+        city_name,
+        _state_name,
+        state_code,
+        _municipality,
+        _municipality_code,
+        _admin_name3,
+        _admin_code3,
+        latitude,
+        longitude,
+        _accuracy
+      ] ->
+        country_code = String.trim(country_code)
+
+        true =
+          :ets.insert(
+            @ets_table_by_lookup,
+            {{country_code, state_code, city_name}, {postal_code, latitude, longitude}}
+          )
+
+        true =
+          :ets.insert(
+            @ets_table_by_id,
+            {postal_code, {country_code, state_code, city_name, latitude, longitude}}
+          )
+
+      [
+        country_code,
+        postal_code,
+        city_name,
+        _state_name,
+        state_code,
+        _municipality,
+        _municipality_code,
+        _admin_name3,
+        _admin_code3,
+        latitude,
+        longitude
+      ] ->
+        true =
+          :ets.insert(
+            @ets_table_by_lookup,
+            {{country_code, state_code, city_name}, {postal_code, latitude, longitude}}
+          )
+
+        true =
+          :ets.insert(
+            @ets_table_by_id,
+            {postal_code, {country_code, state_code, city_name, latitude, longitude}}
+          )
+
+      _data ->
+        :ok
+    end
+  end
+
+  @doc """
+  Finds postal_code information by postal code.
+  """
+  @spec get_postal_code(String.t()) :: %__MODULE__{} | nil
+  def get_postal_code(code) do
+    case :ets.lookup(@ets_table_by_id, code) do
+      [{postal_code, {country_code, state_code, city_name, latitude, longitude}}] ->
+        to_struct(postal_code, country_code, state_code, city_name, latitude, longitude)
+
+      _ ->
+        nil
+    end
+  end
+
+  @doc """
+  Finds postal codes by city code, state code and country code.
+
+  This function returns all postal code founds when the country has multiple
+  cities with the same name.
+  """
+  @spec get_postal_codes(String.t(), String.t(), String.t()) :: %__MODULE__{} | nil
+  def get_postal_codes(country_code, state_code, city_name) do
+    case :ets.lookup(@ets_table_by_lookup, {country_code, state_code, city_name}) do
+      data when is_list(data) ->
+        Enum.map(data, fn x ->
+          {{country_code, state_code, city_name}, {postal_code, latitude, longitude}} = x
+          to_struct(postal_code, country_code, state_code, city_name, latitude, longitude)
+        end)
+
+      _ ->
+        nil
+    end
+  end
+
+  @spec get_postal_codes() :: %__MODULE__{} | nil
+  def get_postal_codes() do
+    :ets.tab2list(@ets_table_by_lookup)
+    |> Enum.map(fn x ->
+      {{country_code, state_code, city_name}, {postal_code, latitude, longitude}} = x
+      to_struct(postal_code, country_code, state_code, city_name, latitude, longitude)
+    end)
+  end
+
+  defp source_file() do
+    default = Application.app_dir(:location, "/priv/postal_codes.csv")
+    Application.get_env(:location, :postal_codes_source_file, default)
+  end
+
+  defp to_struct(postal_code, country_code, state_code, city_name, latitude, longitude) do
+    %__MODULE__{
+      postal_code: postal_code,
+      country_code: country_code,
+      state_code: state_code,
+      city_name: city_name,
+      latitude: latitude,
+      longitude: longitude
+    }
+  end
+end

lib/location/scraper.ex

@@ -0,0 +1,50 @@
+defmodule Location.Scraper do
+  use Tesla
+
+  @version_file Application.app_dir(:location, "/priv/version")
+  @postal_code_url "https://download.geonames.org/export/zip/"
+  @postal_code_dest Application.app_dir(:location, "/priv/")
+
+  def write_date_to_version() do
+    File.write!(@version_file, Date.to_iso8601(Date.utc_today()))
+  end
+
+  def scrape_postal_files() do
+    response = get!(@postal_code_url)
+    {:ok, document} = Floki.parse_document(response.body)
+
+    result =
+      Floki.find(document, "pre")
+
+    result = Floki.find(result, "a") |> Enum.drop(5)
+
+    Enum.map(result, fn x ->
+      [{_, [{_, _href}], [name]}] = Floki.find(x, "a")
+      String.replace(name, ".zip", "")
+    end)
+    |> Enum.join(", ")
+  end
+
+  def fetch_postal_file(file) do
+    response = get!(@postal_code_url <> "#{file}.zip")
+    File.write!(@postal_code_dest <> "/#{file}.zip", response.body)
+  end
+
+  def extract_postal_file(file) do
+    zip_file = Unzip.LocalFile.open("priv/#{file}.zip")
+
+    try do
+      {:ok, unzip} = Unzip.new(zip_file)
+
+      Unzip.file_stream!(unzip, "#{file}.txt")
+      |> Stream.into(File.stream!("priv/#{file}.csv"))
+      |> Stream.run()
+    after
+      Unzip.LocalFile.close(zip_file)
+    end
+  end
+
+  def fetch_postal_files(files) do
+    Enum.each(files, fn file -> fetch_postal_file(file) end)
+  end
+end

lib/location/subdivision.ex

@@ -3,6 +3,10 @@ defmodule Location.Subdivision do
 
   defstruct [:code, :name, :type, :country_code]
 
+  def unload()do
+    :ets.delete(@ets_table)
+  end
+
   def load() do
     ets = :ets.new(@ets_table, [:named_table])
 

mix_tasks/update_english_translations.ex → lib/mix/tasks/update_english_translations.ex

@@ -1,5 +1,6 @@
-defmodule Mix.Tasks.UpdateEnglishTranslations do
+defmodule Mix.Tasks.Location.UpdateEnglishTranslations do
   use Mix.Task
+  @shortdoc "Updates english translations for locations"
 
   @cldr_url "https://raw.githubusercontent.com/unicode-org/cldr/main/common/subdivisions/en.xml"
   @translations_dest Application.app_dir(:location, "/priv/iso_3166-2.en-translations.json")