From b1010eb7edcd2d9ccfeb53c8765199e9a977b590 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Sun, 29 Mar 2020 16:38:19 +0700 Subject: [PATCH 01/11] Add keywords table and rename page controller to dashboard --- lib/google_crawler/search.ex | 61 +++++++++++++++++++ lib/google_crawler/search/keyword.ex | 16 +++++ ..._controller.ex => dashboard_controller.ex} | 2 +- .../controllers/registration_controller.ex | 2 +- .../controllers/session_controller.ex | 2 +- .../plugs/skip_after_auth.ex | 2 +- lib/google_crawler_web/router.ex | 3 +- .../{page => dashboard}/index.html.eex | 0 .../views/dashboard_view.ex | 3 + lib/google_crawler_web/views/page_view.ex | 3 - .../20200329085134_create_keywords.exs | 12 ++++ test/google_crawler/search_test.exs | 40 ++++++++++++ ...test.exs => dashboard_controller_test.exs} | 6 +- .../registration_controller_test.exs | 2 +- .../controllers/session_controller_test.exs | 4 +- 15 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 lib/google_crawler/search.ex create mode 100644 lib/google_crawler/search/keyword.ex rename lib/google_crawler_web/controllers/{page_controller.ex => dashboard_controller.ex} (68%) rename lib/google_crawler_web/templates/{page => dashboard}/index.html.eex (100%) create mode 100644 lib/google_crawler_web/views/dashboard_view.ex delete mode 100644 lib/google_crawler_web/views/page_view.ex create mode 100644 priv/repo/migrations/20200329085134_create_keywords.exs create mode 100644 test/google_crawler/search_test.exs rename test/google_crawler_web/controllers/{page_controller_test.exs => dashboard_controller_test.exs} (78%) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex new file mode 100644 index 0000000..b7e3ce8 --- /dev/null +++ b/lib/google_crawler/search.ex @@ -0,0 +1,61 @@ +defmodule GoogleCrawler.Search do + @moduledoc """ + The Search context. + """ + + import Ecto.Query, warn: false + alias GoogleCrawler.Repo + + alias GoogleCrawler.Search.Keyword + + @doc """ + Returns the list of keywords. + + ## Examples + + iex> list_keywords() + [%Keyword{}, ...] + + """ + def list_keywords do + Repo.all(Keyword) + end + + @doc """ + Gets a single keyword. + + Raises `Ecto.NoResultsError` if the Keyword does not exist. + + ## Examples + + iex> get_keyword!(123) + %Keyword{} + + iex> get_keyword!(456) + ** (Ecto.NoResultsError) + + """ + def get_keyword(id), do: Repo.get(Keyword, id) + + @doc """ + Creates a keyword. + + ## Examples + + iex> create_keyword(%{field: value}) + {:ok, %Keyword{}} + + iex> create_keyword(%{field: bad_value}) + {:error, %Ecto.Changeset{}} + + """ + def create_keyword(attrs \\ %{}) do + %Keyword{} + |> Keyword.changeset(attrs) + |> Repo.insert() + end + + def parse_keywords_from_csv do + + end +end diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex new file mode 100644 index 0000000..99d437e --- /dev/null +++ b/lib/google_crawler/search/keyword.ex @@ -0,0 +1,16 @@ +defmodule GoogleCrawler.Search.Keyword do + use Ecto.Schema + import Ecto.Changeset + + schema "keywords" do + field :keyword, :string + + timestamps() + end + + def changeset(keyword, attrs) do + keyword + |> cast(attrs, [:keyword]) + |> validate_required([:keyword]) + end +end diff --git a/lib/google_crawler_web/controllers/page_controller.ex b/lib/google_crawler_web/controllers/dashboard_controller.ex similarity index 68% rename from lib/google_crawler_web/controllers/page_controller.ex rename to lib/google_crawler_web/controllers/dashboard_controller.ex index 92121e0..5606c62 100644 --- a/lib/google_crawler_web/controllers/page_controller.ex +++ b/lib/google_crawler_web/controllers/dashboard_controller.ex @@ -1,4 +1,4 @@ -defmodule GoogleCrawlerWeb.PageController do +defmodule GoogleCrawlerWeb.DashboardController do use GoogleCrawlerWeb, :controller def index(conn, _params) do diff --git a/lib/google_crawler_web/controllers/registration_controller.ex b/lib/google_crawler_web/controllers/registration_controller.ex index 435cb0e..e7f8d2e 100644 --- a/lib/google_crawler_web/controllers/registration_controller.ex +++ b/lib/google_crawler_web/controllers/registration_controller.ex @@ -16,7 +16,7 @@ defmodule GoogleCrawlerWeb.RegistrationController do conn |> put_flash(:info, gettext("You have signed up successfully!")) # TODO: Change to login path - |> redirect(to: Routes.page_path(conn, :index)) + |> redirect(to: Routes.dashboard_path(conn, :index)) {:error, changeset} -> render(conn, "new.html", changeset: changeset) diff --git a/lib/google_crawler_web/controllers/session_controller.ex b/lib/google_crawler_web/controllers/session_controller.ex index dde4e80..896e290 100644 --- a/lib/google_crawler_web/controllers/session_controller.ex +++ b/lib/google_crawler_web/controllers/session_controller.ex @@ -16,7 +16,7 @@ defmodule GoogleCrawlerWeb.SessionController do conn |> put_flash(:info, gettext("Welcome back")) |> put_session(:current_user_id, user.id) - |> redirect(to: Routes.page_path(conn, :index)) + |> redirect(to: Routes.dashboard_path(conn, :index)) {:error, _reason} -> conn diff --git a/lib/google_crawler_web/plugs/skip_after_auth.ex b/lib/google_crawler_web/plugs/skip_after_auth.ex index a88f1ea..739ea8f 100644 --- a/lib/google_crawler_web/plugs/skip_after_auth.ex +++ b/lib/google_crawler_web/plugs/skip_after_auth.ex @@ -17,7 +17,7 @@ defmodule GoogleCrawlerWeb.Plugs.SkipAfterAuth do if conn.assigns.user_signed_in? do conn |> put_flash(:info, gettext("You are already signed in.")) - |> redirect(to: Routes.page_path(conn, :index)) + |> redirect(to: Routes.dashboard_path(conn, :index)) |> halt() else conn diff --git a/lib/google_crawler_web/router.ex b/lib/google_crawler_web/router.ex index 8d13902..e2420f9 100644 --- a/lib/google_crawler_web/router.ex +++ b/lib/google_crawler_web/router.ex @@ -28,8 +28,7 @@ defmodule GoogleCrawlerWeb.Router do resources "/sessions", SessionController, only: [:delete] - # TODO: Cleanup this default route - get "/", PageController, :index + get "/", DashboardController, :index end # Other scopes may use custom stacks. diff --git a/lib/google_crawler_web/templates/page/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex similarity index 100% rename from lib/google_crawler_web/templates/page/index.html.eex rename to lib/google_crawler_web/templates/dashboard/index.html.eex diff --git a/lib/google_crawler_web/views/dashboard_view.ex b/lib/google_crawler_web/views/dashboard_view.ex new file mode 100644 index 0000000..0b485a2 --- /dev/null +++ b/lib/google_crawler_web/views/dashboard_view.ex @@ -0,0 +1,3 @@ +defmodule GoogleCrawlerWeb.DashboardView do + use GoogleCrawlerWeb, :view +end diff --git a/lib/google_crawler_web/views/page_view.ex b/lib/google_crawler_web/views/page_view.ex deleted file mode 100644 index 765f665..0000000 --- a/lib/google_crawler_web/views/page_view.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule GoogleCrawlerWeb.PageView do - use GoogleCrawlerWeb, :view -end diff --git a/priv/repo/migrations/20200329085134_create_keywords.exs b/priv/repo/migrations/20200329085134_create_keywords.exs new file mode 100644 index 0000000..28b4691 --- /dev/null +++ b/priv/repo/migrations/20200329085134_create_keywords.exs @@ -0,0 +1,12 @@ +defmodule GoogleCrawler.Repo.Migrations.CreateKeywords do + use Ecto.Migration + + def change do + create table(:keywords) do + add :keyword, :string + + timestamps() + end + + end +end diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs new file mode 100644 index 0000000..06f6154 --- /dev/null +++ b/test/google_crawler/search_test.exs @@ -0,0 +1,40 @@ +defmodule GoogleCrawler.SearchTest do + use GoogleCrawler.DataCase + + alias GoogleCrawler.Search + + describe "keywords" do + alias GoogleCrawler.Search.Keyword + + @valid_attrs %{keyword: "some keyword"} + @invalid_attrs %{keyword: nil} + + def keyword_fixture(attrs \\ %{}) do + {:ok, keyword} = + attrs + |> Enum.into(@valid_attrs) + |> Search.create_keyword() + + keyword + end + + test "list_keywords/0 returns all keywords" do + keyword = keyword_fixture() + assert Search.list_keywords() == [keyword] + end + + test "get_keyword/1 returns the keyword with given id" do + keyword = keyword_fixture() + assert Search.get_keyword(keyword.id) == keyword + end + + test "create_keyword/1 with valid data creates a keyword" do + assert {:ok, %Keyword{} = keyword} = Search.create_keyword(@valid_attrs) + assert keyword.keyword == "some keyword" + end + + test "create_keyword/1 with invalid data returns error changeset" do + assert {:error, %Ecto.Changeset{}} = Search.create_keyword(@invalid_attrs) + end + end +end diff --git a/test/google_crawler_web/controllers/page_controller_test.exs b/test/google_crawler_web/controllers/dashboard_controller_test.exs similarity index 78% rename from test/google_crawler_web/controllers/page_controller_test.exs rename to test/google_crawler_web/controllers/dashboard_controller_test.exs index e6161e9..50b2baf 100644 --- a/test/google_crawler_web/controllers/page_controller_test.exs +++ b/test/google_crawler_web/controllers/dashboard_controller_test.exs @@ -1,4 +1,4 @@ -defmodule GoogleCrawlerWeb.PageControllerTest do +defmodule GoogleCrawlerWeb.DashboardControllerTest do use GoogleCrawlerWeb.ConnCase alias GoogleCrawler.UserFactory @@ -10,7 +10,7 @@ defmodule GoogleCrawlerWeb.PageControllerTest do conn |> init_test_session(%{}) |> put_session(:current_user_id, user.id) - |> get(Routes.page_path(conn, :index)) + |> get(Routes.dashboard_path(conn, :index)) assert html_response(conn, 200) =~ "Welcome to Phoenix!" end @@ -19,7 +19,7 @@ defmodule GoogleCrawlerWeb.PageControllerTest do conn = conn |> init_test_session(%{}) - |> get(Routes.page_path(conn, :index)) + |> get(Routes.dashboard_path(conn, :index)) assert redirected_to(conn) == Routes.session_path(conn, :new) end diff --git a/test/google_crawler_web/controllers/registration_controller_test.exs b/test/google_crawler_web/controllers/registration_controller_test.exs index 5d4238c..feadbe0 100644 --- a/test/google_crawler_web/controllers/registration_controller_test.exs +++ b/test/google_crawler_web/controllers/registration_controller_test.exs @@ -27,7 +27,7 @@ defmodule GoogleCrawlerWeb.RegistrationControllerTest do conn = post(conn, Routes.registration_path(conn, :create, user: user_attrs)) - assert redirected_to(conn) == Routes.page_path(conn, :index) + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) assert get_flash(conn, :info) == "You have signed up successfully!" end diff --git a/test/google_crawler_web/controllers/session_controller_test.exs b/test/google_crawler_web/controllers/session_controller_test.exs index 6b42a82..1cc9022 100644 --- a/test/google_crawler_web/controllers/session_controller_test.exs +++ b/test/google_crawler_web/controllers/session_controller_test.exs @@ -18,7 +18,7 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do |> put_session(:current_user_id, user.id) |> get(Routes.session_path(conn, :new)) - assert redirected_to(conn) == Routes.page_path(conn, :index) + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end describe "create/2" do @@ -30,7 +30,7 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do user: %{email: user.email, password: user.password} ) - assert redirected_to(conn) == Routes.page_path(conn, :index) + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) assert get_flash(conn, :info) == "Welcome back" assert get_session(conn, :current_user_id) == user.id end From 456bff45bc660366cfdd27610e6dce3d6f33aa98 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Sun, 29 Mar 2020 21:28:32 +0700 Subject: [PATCH 02/11] Implement the upload form and controller --- lib/google_crawler/search.ex | 1 - lib/google_crawler/search/keyword.ex | 2 +- lib/google_crawler/search/keyword_file.ex | 27 ++++++++++++++ .../controllers/dashboard_controller.ex | 6 +++- .../controllers/upload_controller.ex | 20 +++++++++++ lib/google_crawler_web/router.ex | 1 + .../templates/dashboard/index.html.eex | 36 ++----------------- .../templates/keyword/_form.html.eex | 16 +++++++++ lib/google_crawler_web/views/keyword_view.ex | 3 ++ .../20200329085134_create_keywords.exs | 1 - 10 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 lib/google_crawler/search/keyword_file.ex create mode 100644 lib/google_crawler_web/controllers/upload_controller.ex create mode 100644 lib/google_crawler_web/templates/keyword/_form.html.eex create mode 100644 lib/google_crawler_web/views/keyword_view.ex diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index b7e3ce8..d60ad8e 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -56,6 +56,5 @@ defmodule GoogleCrawler.Search do end def parse_keywords_from_csv do - end end diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 99d437e..50656c7 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -8,7 +8,7 @@ defmodule GoogleCrawler.Search.Keyword do timestamps() end - def changeset(keyword, attrs) do + def changeset(keyword, attrs \\ %{}) do keyword |> cast(attrs, [:keyword]) |> validate_required([:keyword]) diff --git a/lib/google_crawler/search/keyword_file.ex b/lib/google_crawler/search/keyword_file.ex new file mode 100644 index 0000000..20c0b87 --- /dev/null +++ b/lib/google_crawler/search/keyword_file.ex @@ -0,0 +1,27 @@ +defmodule GoogleCrawler.Search.KeywordFile do + use Ecto.Schema + import Ecto.Changeset + + embedded_schema do + field :file, :map + end + + @accept_file_ext ~w(.csv) + + def changeset(keyword_file, attrs \\ %{}) do + keyword_file + |> cast(attrs, [:file]) + |> validate_required([:file]) + |> validate_file_ext() + end + + defp validate_file_ext(changeset) do + validate_change(changeset, :file, fn :file, file -> + if Enum.member?(@accept_file_ext, Path.extname(file.filename)) do + [] + else + [file: "is not supported"] + end + end) + end +end diff --git a/lib/google_crawler_web/controllers/dashboard_controller.ex b/lib/google_crawler_web/controllers/dashboard_controller.ex index 5606c62..e341dbb 100644 --- a/lib/google_crawler_web/controllers/dashboard_controller.ex +++ b/lib/google_crawler_web/controllers/dashboard_controller.ex @@ -1,7 +1,11 @@ defmodule GoogleCrawlerWeb.DashboardController do use GoogleCrawlerWeb, :controller + alias GoogleCrawler.Search.KeywordFile + def index(conn, _params) do - render(conn, "index.html") + changeset = KeywordFile.changeset(%KeywordFile{}) + + render(conn, "index.html", changeset: changeset) end end diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex new file mode 100644 index 0000000..6737ed6 --- /dev/null +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -0,0 +1,20 @@ +defmodule GoogleCrawlerWeb.UploadController do + use GoogleCrawlerWeb, :controller + import Ecto.Changeset, only: [get_change: 3] + + alias GoogleCrawler.Search.KeywordFile + + def create(conn, %{"keyword_file" => keyword_file}) do + changeset = KeywordFile.changeset(%KeywordFile{}, keyword_file) + + if changeset.valid? do + # TODO: Parse CSV + file = get_change(changeset, :file, nil) + file.path + else + conn + |> put_flash(:error, gettext("Invalid file, please select again.")) + |> redirect(to: Routes.dashboard_path(conn, :index)) + end + end +end diff --git a/lib/google_crawler_web/router.ex b/lib/google_crawler_web/router.ex index e2420f9..7145612 100644 --- a/lib/google_crawler_web/router.ex +++ b/lib/google_crawler_web/router.ex @@ -27,6 +27,7 @@ defmodule GoogleCrawlerWeb.Router do pipe_through [:browser, GoogleCrawlerWeb.Plugs.EnsureAuth] resources "/sessions", SessionController, only: [:delete] + resources "/upload", UploadController, only: [:create] get "/", DashboardController, :index end diff --git a/lib/google_crawler_web/templates/dashboard/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex index 8cbd9d8..160d269 100644 --- a/lib/google_crawler_web/templates/dashboard/index.html.eex +++ b/lib/google_crawler_web/templates/dashboard/index.html.eex @@ -1,35 +1,3 @@ -
-

<%= gettext "Welcome to %{name}!", name: "Phoenix" %>

-

A productive web framework that
does not compromise speed or maintainability.

-
+<% IO.inspect assigns %> -
- - -
+<%= render GoogleCrawlerWeb.KeywordView, "_form.html", assigns %> diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex new file mode 100644 index 0000000..87060a0 --- /dev/null +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -0,0 +1,16 @@ +
+

<%= gettext("Upload your keyword file (.csv)") %>

+ <%= form_for @changeset, Routes.upload_path(@conn, :create), [multipart: true], fn f -> %> + <%= label f, :file %> + <%= file_input f, :file, required: true, accept: "text/csv" %> + <%= error_tag f, :file %> + + <%= submit gettext("Upload") %> + <% end %> +
+
+
+

<%= gettext("Keywords") %>

+

<%= gettext("You don't have any keywords.") %>

+
+ diff --git a/lib/google_crawler_web/views/keyword_view.ex b/lib/google_crawler_web/views/keyword_view.ex new file mode 100644 index 0000000..95f6c5e --- /dev/null +++ b/lib/google_crawler_web/views/keyword_view.ex @@ -0,0 +1,3 @@ +defmodule GoogleCrawlerWeb.KeywordView do + use GoogleCrawlerWeb, :view +end diff --git a/priv/repo/migrations/20200329085134_create_keywords.exs b/priv/repo/migrations/20200329085134_create_keywords.exs index 28b4691..f86763a 100644 --- a/priv/repo/migrations/20200329085134_create_keywords.exs +++ b/priv/repo/migrations/20200329085134_create_keywords.exs @@ -7,6 +7,5 @@ defmodule GoogleCrawler.Repo.Migrations.CreateKeywords do timestamps() end - end end From f020d47b2e743a0062e601d4796706cf4594d9d8 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 09:22:20 +0700 Subject: [PATCH 03/11] Parse keyword csv file --- lib/google_crawler/search.ex | 4 +++- lib/google_crawler/search/keyword_file.ex | 10 ++++++++++ .../controllers/upload_controller.ex | 4 ++-- mix.exs | 3 ++- mix.lock | 2 ++ 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index d60ad8e..b93526f 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -7,6 +7,7 @@ defmodule GoogleCrawler.Search do alias GoogleCrawler.Repo alias GoogleCrawler.Search.Keyword + alias GoogleCrawler.Search.KeywordFile @doc """ Returns the list of keywords. @@ -55,6 +56,7 @@ defmodule GoogleCrawler.Search do |> Repo.insert() end - def parse_keywords_from_csv do + def parse_keywords_from_file!(file_path, content_type) do + KeywordFile.parse!(file_path, content_type) end end diff --git a/lib/google_crawler/search/keyword_file.ex b/lib/google_crawler/search/keyword_file.ex index 20c0b87..243b4b0 100644 --- a/lib/google_crawler/search/keyword_file.ex +++ b/lib/google_crawler/search/keyword_file.ex @@ -15,6 +15,16 @@ defmodule GoogleCrawler.Search.KeywordFile do |> validate_file_ext() end + def parse!(file_path, "text/csv") do + file_path + |> File.stream!() + |> CSV.decode() + end + + def parse!(_file_path, _unexpected_content_type) do + raise "File with this extenstion is not supported" + end + defp validate_file_ext(changeset) do validate_change(changeset, :file, fn :file, file -> if Enum.member?(@accept_file_ext, Path.extname(file.filename)) do diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index 6737ed6..6f633db 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -2,15 +2,15 @@ defmodule GoogleCrawlerWeb.UploadController do use GoogleCrawlerWeb, :controller import Ecto.Changeset, only: [get_change: 3] + alias GoogleCrawler.Search alias GoogleCrawler.Search.KeywordFile def create(conn, %{"keyword_file" => keyword_file}) do changeset = KeywordFile.changeset(%KeywordFile{}, keyword_file) if changeset.valid? do - # TODO: Parse CSV file = get_change(changeset, :file, nil) - file.path + Search.parse_keywords_from_file!(file.path, file.content_type) else conn |> put_flash(:error, gettext("Invalid file, please select again.")) diff --git a/mix.exs b/mix.exs index 736f017..5bb4cfb 100644 --- a/mix.exs +++ b/mix.exs @@ -44,7 +44,8 @@ defmodule GoogleCrawler.MixProject do {:jason, "~> 1.0"}, {:plug_cowboy, "~> 2.0"}, {:bcrypt_elixir, "~> 2.0"}, - {:faker_elixir_octopus, "~> 1.0.0", only: [:dev, :test]} + {:faker_elixir_octopus, "~> 1.0.0", only: [:dev, :test]}, + {:csv, "~> 2.3"} ] end diff --git a/mix.lock b/mix.lock index 1f2e5fe..6489f6e 100644 --- a/mix.lock +++ b/mix.lock @@ -4,6 +4,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "04fd8c6a39edc6aaa9c26123009200fc61f92a3a94f3178c527b70b767c6e605"}, "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, + "csv": {:hex, :csv, "2.3.1", "9ce11eff5a74a07baf3787b2b19dd798724d29a9c3a492a41df39f6af686da0e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "86626e1c89a4ad9a96d0d9c638f9e88c2346b89b4ba1611988594ebe72b5d5ee"}, "db_connection": {:hex, :db_connection, "2.2.1", "caee17725495f5129cb7faebde001dc4406796f12a62b8949f4ac69315080566", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "2b02ece62d9f983fcd40954e443b7d9e6589664380e5546b2b9b523cd0fb59e1"}, "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, "ecto": {:hex, :ecto, "3.3.4", "95b05c82ae91361475e5491c9f3ac47632f940b3f92ae3988ac1aad04989c5bb", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "9b96cbb83a94713731461ea48521b178b0e3863d310a39a3948c807266eebd69"}, @@ -15,6 +16,7 @@ "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, + "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm", "639b2e8749e11b87b9eb42f2ad325d161c170b39b288ac8d04c4f31f8f0823eb"}, "phoenix": {:hex, :phoenix, "1.4.16", "2cbbe0c81e6601567c44cc380c33aa42a1372ac1426e3de3d93ac448a7ec4308", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "856cc1a032fa53822737413cf51aa60e750525d7ece7d1c0576d90d7c0f05c24"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"}, "phoenix_html": {:hex, :phoenix_html, "2.14.1", "7dabafadedb552db142aacbd1f11de1c0bbaa247f90c449ca549d5e30bbc66b4", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "536d5200ad37fecfe55b3241d90b7a8c3a2ca60cd012fc065f776324fa9ab0a9"}, From 2a073c68f21651087d210565e51223e6ee4a9ea4 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 10:34:20 +0700 Subject: [PATCH 04/11] Fix tests and remove debugging from dashboard --- lib/google_crawler_web/templates/dashboard/index.html.eex | 2 -- .../controllers/dashboard_controller_test.exs | 2 +- .../controllers/registration_controller_test.exs | 4 ++-- test/google_crawler_web/plugs/skip_after_auth_test.exs | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/google_crawler_web/templates/dashboard/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex index 160d269..385bc64 100644 --- a/lib/google_crawler_web/templates/dashboard/index.html.eex +++ b/lib/google_crawler_web/templates/dashboard/index.html.eex @@ -1,3 +1 @@ -<% IO.inspect assigns %> - <%= render GoogleCrawlerWeb.KeywordView, "_form.html", assigns %> diff --git a/test/google_crawler_web/controllers/dashboard_controller_test.exs b/test/google_crawler_web/controllers/dashboard_controller_test.exs index 50b2baf..2cc73bc 100644 --- a/test/google_crawler_web/controllers/dashboard_controller_test.exs +++ b/test/google_crawler_web/controllers/dashboard_controller_test.exs @@ -12,7 +12,7 @@ defmodule GoogleCrawlerWeb.DashboardControllerTest do |> put_session(:current_user_id, user.id) |> get(Routes.dashboard_path(conn, :index)) - assert html_response(conn, 200) =~ "Welcome to Phoenix!" + assert html_response(conn, 200) =~ "Upload your keyword file" end test "GET / redirects to the login page if the user has not logged in", %{conn: conn} do diff --git a/test/google_crawler_web/controllers/registration_controller_test.exs b/test/google_crawler_web/controllers/registration_controller_test.exs index feadbe0..48bbd9c 100644 --- a/test/google_crawler_web/controllers/registration_controller_test.exs +++ b/test/google_crawler_web/controllers/registration_controller_test.exs @@ -9,7 +9,7 @@ defmodule GoogleCrawlerWeb.RegistrationControllerTest do assert html_response(conn, 200) =~ "Create Account" end - test "new/2 redirects to the pages controller if the user has already logged in", %{conn: conn} do + test "new/2 redirects to the user dashboard if the user has already logged in", %{conn: conn} do user = UserFactory.create() conn = @@ -18,7 +18,7 @@ defmodule GoogleCrawlerWeb.RegistrationControllerTest do |> put_session(:current_user_id, user.id) |> get(Routes.registration_path(conn, :new)) - assert redirected_to(conn) == Routes.page_path(conn, :index) + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end describe "create/2" do diff --git a/test/google_crawler_web/plugs/skip_after_auth_test.exs b/test/google_crawler_web/plugs/skip_after_auth_test.exs index 46590bf..8f381c3 100644 --- a/test/google_crawler_web/plugs/skip_after_auth_test.exs +++ b/test/google_crawler_web/plugs/skip_after_auth_test.exs @@ -4,7 +4,7 @@ defmodule GoogleCrawlerWeb.SkipAfterAuthTest do alias GoogleCrawler.UserFactory - test "redirects to the index page if the user has already logged in" do + test "redirects to the user dashboard if the user has already logged in" do user = UserFactory.create() conn = @@ -15,7 +15,7 @@ defmodule GoogleCrawlerWeb.SkipAfterAuthTest do |> call(%{}) assert conn.halted - assert redirected_to(conn) == Routes.page_path(conn, :index) + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) assert get_flash(conn, :info) == "You are already signed in." end From 1cca948747a54bc370d21791f1b930a24800871a Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 11:03:53 +0700 Subject: [PATCH 05/11] Add docs and add simple text render from upload controller --- lib/google_crawler/search.ex | 11 +++++++++++ lib/google_crawler/search/keyword_file.ex | 2 +- .../controllers/upload_controller.ex | 5 ++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index b93526f..e10a31f 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -56,6 +56,17 @@ defmodule GoogleCrawler.Search do |> Repo.insert() end + @doc """ + Parses the keyword from the given file. + Returns the stream for each line in the csv file as {ok: result}. + Raise an exception if the file content type is not supported or the file parsing is failed. + + ### Examples + + iex > parse_keywords_from_file!("var/folder/abcdef", "text/csv") |> Enum.to_list + [ok: ["hotels"], ok: ["restaurants"]] + + """ def parse_keywords_from_file!(file_path, content_type) do KeywordFile.parse!(file_path, content_type) end diff --git a/lib/google_crawler/search/keyword_file.ex b/lib/google_crawler/search/keyword_file.ex index 243b4b0..1bbf5f8 100644 --- a/lib/google_crawler/search/keyword_file.ex +++ b/lib/google_crawler/search/keyword_file.ex @@ -18,7 +18,7 @@ defmodule GoogleCrawler.Search.KeywordFile do def parse!(file_path, "text/csv") do file_path |> File.stream!() - |> CSV.decode() + |> CSV.decode!() end def parse!(_file_path, _unexpected_content_type) do diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index 6f633db..9886309 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -10,7 +10,10 @@ defmodule GoogleCrawlerWeb.UploadController do if changeset.valid? do file = get_change(changeset, :file, nil) - Search.parse_keywords_from_file!(file.path, file.content_type) + result = Search.parse_keywords_from_file!(file.path, file.content_type) + + # TODO: Save these keywords and triggers the task to google search for each keyword + text(conn, result |> Enum.map(fn {:ok, [keyword]} -> keyword end) |> Enum.join(", ")) else conn |> put_flash(:error, gettext("Invalid file, please select again.")) From d725d0099b8a64e7a293865886e6929593304a1b Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 11:28:47 +0700 Subject: [PATCH 06/11] Add authentication conn helper for conn case --- .../templates/keyword/_form.html.eex | 1 + .../keyword_files/invalid_keyword.csv | 1 + .../keyword_files/unsupported_keyword.txt | 1 + test/fixtures/keyword_files/valid_keyword.csv | 3 ++ .../controllers/dashboard_controller_test.exs | 7 +-- .../registration_controller_test.exs | 4 +- .../controllers/session_controller_test.exs | 46 ++++++++----------- .../plugs/ensure_auth_test.exs | 4 +- .../plugs/set_current_user_test.exs | 3 +- .../plugs/skip_after_auth_test.exs | 4 +- test/support/conn_case.ex | 6 +++ 11 files changed, 38 insertions(+), 42 deletions(-) create mode 100644 test/fixtures/keyword_files/invalid_keyword.csv create mode 100644 test/fixtures/keyword_files/unsupported_keyword.txt create mode 100644 test/fixtures/keyword_files/valid_keyword.csv diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex index 87060a0..4d54c70 100644 --- a/lib/google_crawler_web/templates/keyword/_form.html.eex +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -1,5 +1,6 @@

<%= gettext("Upload your keyword file (.csv)") %>

+

<%= gettext("📝 Please put one keyword per line") %>

<%= form_for @changeset, Routes.upload_path(@conn, :create), [multipart: true], fn f -> %> <%= label f, :file %> <%= file_input f, :file, required: true, accept: "text/csv" %> diff --git a/test/fixtures/keyword_files/invalid_keyword.csv b/test/fixtures/keyword_files/invalid_keyword.csv new file mode 100644 index 0000000..e7b0d58 --- /dev/null +++ b/test/fixtures/keyword_files/invalid_keyword.csv @@ -0,0 +1 @@ +elixir ruby javascript diff --git a/test/fixtures/keyword_files/unsupported_keyword.txt b/test/fixtures/keyword_files/unsupported_keyword.txt new file mode 100644 index 0000000..e7b0d58 --- /dev/null +++ b/test/fixtures/keyword_files/unsupported_keyword.txt @@ -0,0 +1 @@ +elixir ruby javascript diff --git a/test/fixtures/keyword_files/valid_keyword.csv b/test/fixtures/keyword_files/valid_keyword.csv new file mode 100644 index 0000000..cefeaf5 --- /dev/null +++ b/test/fixtures/keyword_files/valid_keyword.csv @@ -0,0 +1,3 @@ +elixir +ruby +javascript diff --git a/test/google_crawler_web/controllers/dashboard_controller_test.exs b/test/google_crawler_web/controllers/dashboard_controller_test.exs index 2cc73bc..f03e68f 100644 --- a/test/google_crawler_web/controllers/dashboard_controller_test.exs +++ b/test/google_crawler_web/controllers/dashboard_controller_test.exs @@ -3,13 +3,11 @@ defmodule GoogleCrawlerWeb.DashboardControllerTest do alias GoogleCrawler.UserFactory - test "GET / renders the index template", %{conn: conn} do + test "GET / renders the index template with upload keyword form", %{conn: conn} do user = UserFactory.create() conn = - conn - |> init_test_session(%{}) - |> put_session(:current_user_id, user.id) + build_authenticated_conn(user) |> get(Routes.dashboard_path(conn, :index)) assert html_response(conn, 200) =~ "Upload your keyword file" @@ -18,7 +16,6 @@ defmodule GoogleCrawlerWeb.DashboardControllerTest do test "GET / redirects to the login page if the user has not logged in", %{conn: conn} do conn = conn - |> init_test_session(%{}) |> get(Routes.dashboard_path(conn, :index)) assert redirected_to(conn) == Routes.session_path(conn, :new) diff --git a/test/google_crawler_web/controllers/registration_controller_test.exs b/test/google_crawler_web/controllers/registration_controller_test.exs index 48bbd9c..e08e789 100644 --- a/test/google_crawler_web/controllers/registration_controller_test.exs +++ b/test/google_crawler_web/controllers/registration_controller_test.exs @@ -13,9 +13,7 @@ defmodule GoogleCrawlerWeb.RegistrationControllerTest do user = UserFactory.create() conn = - conn - |> init_test_session(%{}) - |> put_session(:current_user_id, user.id) + build_authenticated_conn(user) |> get(Routes.registration_path(conn, :new)) assert redirected_to(conn) == Routes.dashboard_path(conn, :index) diff --git a/test/google_crawler_web/controllers/session_controller_test.exs b/test/google_crawler_web/controllers/session_controller_test.exs index 1cc9022..d553aa6 100644 --- a/test/google_crawler_web/controllers/session_controller_test.exs +++ b/test/google_crawler_web/controllers/session_controller_test.exs @@ -13,48 +13,42 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do user = UserFactory.create() conn = - conn - |> init_test_session(%{}) - |> put_session(:current_user_id, user.id) + build_authenticated_conn(user) |> get(Routes.session_path(conn, :new)) assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end - describe "create/2" do - test "redirects to page controller when user credentials are valid", %{conn: conn} do - user = UserFactory.create() + test "create/2 redirects to page controller when user credentials are valid", %{conn: conn} do + user = UserFactory.create() - conn = - post(conn, Routes.session_path(conn, :create), - user: %{email: user.email, password: user.password} - ) + conn = + post(conn, Routes.session_path(conn, :create), + user: %{email: user.email, password: user.password} + ) - assert redirected_to(conn) == Routes.dashboard_path(conn, :index) - assert get_flash(conn, :info) == "Welcome back" - assert get_session(conn, :current_user_id) == user.id - end + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) + assert get_flash(conn, :info) == "Welcome back" + assert get_session(conn, :current_user_id) == user.id + end - test "renders the error when user credentials are invalid", %{conn: conn} do - user = UserFactory.create() + test "create/2 renders the error when user credentials are invalid", %{conn: conn} do + user = UserFactory.create() - conn = - post(conn, Routes.session_path(conn, :create), - user: %{email: user.email, password: "invalid_password"} - ) + conn = + post(conn, Routes.session_path(conn, :create), + user: %{email: user.email, password: "invalid_password"} + ) - assert redirected_to(conn) == Routes.session_path(conn, :new) - assert get_flash(conn, :error) == "The email or password is incorrect, please try again" - end + assert redirected_to(conn) == Routes.session_path(conn, :new) + assert get_flash(conn, :error) == "The email or password is incorrect, please try again" end test "delete/2 clears the user session and redirects to the log in page", %{conn: conn} do user = UserFactory.create() conn = - conn - |> init_test_session(%{}) - |> put_session(:current_user_id, user.id) + build_authenticated_conn(user) |> delete(Routes.session_path(conn, :delete, user)) assert get_session(conn, :current_user_id) == nil diff --git a/test/google_crawler_web/plugs/ensure_auth_test.exs b/test/google_crawler_web/plugs/ensure_auth_test.exs index cc5049d..dc55631 100644 --- a/test/google_crawler_web/plugs/ensure_auth_test.exs +++ b/test/google_crawler_web/plugs/ensure_auth_test.exs @@ -8,9 +8,7 @@ defmodule GoogleCrawlerWeb.EnsureAuthTest do user = UserFactory.create() conn = - build_conn() - |> init_test_session(current_user_id: user.id) - |> assign(:user_signed_in?, true) + build_authenticated_conn(user) |> call(%{}) refute conn.halted diff --git a/test/google_crawler_web/plugs/set_current_user_test.exs b/test/google_crawler_web/plugs/set_current_user_test.exs index 35819c5..1886814 100644 --- a/test/google_crawler_web/plugs/set_current_user_test.exs +++ b/test/google_crawler_web/plugs/set_current_user_test.exs @@ -8,8 +8,7 @@ defmodule GoogleCrawlerWeb.SetCurrentUserTest do user = UserFactory.create() conn = - build_conn() - |> init_test_session(current_user_id: user.id) + build_authenticated_conn(user) |> call(%{}) assert conn.assigns.current_user.id == user.id diff --git a/test/google_crawler_web/plugs/skip_after_auth_test.exs b/test/google_crawler_web/plugs/skip_after_auth_test.exs index 8f381c3..459db66 100644 --- a/test/google_crawler_web/plugs/skip_after_auth_test.exs +++ b/test/google_crawler_web/plugs/skip_after_auth_test.exs @@ -8,10 +8,8 @@ defmodule GoogleCrawlerWeb.SkipAfterAuthTest do user = UserFactory.create() conn = - build_conn() - |> init_test_session(current_user_id: user.id) + build_authenticated_conn(user) |> fetch_flash - |> assign(:user_signed_in?, true) |> call(%{}) assert conn.halted diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index 05e29ce..af89bd3 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -26,6 +26,12 @@ defmodule GoogleCrawlerWeb.ConnCase do # The default endpoint for testing @endpoint GoogleCrawlerWeb.Endpoint + + def build_authenticated_conn(user) do + build_conn() + |> Plug.Test.init_test_session(current_user_id: user.id) + |> assign(:user_signed_in?, true) + end end end From df5cc146205174ffd4a79d113793a53b00a158ae Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 14:00:42 +0700 Subject: [PATCH 07/11] Add tests for upload controller --- lib/google_crawler/search.ex | 2 +- .../controllers/upload_controller.ex | 2 +- .../templates/keyword/_form.html.eex | 2 +- .../keyword_files/invalid_keyword.csv | 3 +- .../registration_controller_test.exs | 22 ++++---- .../controllers/session_controller_test.exs | 6 +-- .../controllers/upload_controller_test.exs | 50 +++++++++++++++++++ 7 files changed, 68 insertions(+), 19 deletions(-) create mode 100644 test/google_crawler_web/controllers/upload_controller_test.exs diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index e10a31f..6c3b7bc 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -58,7 +58,7 @@ defmodule GoogleCrawler.Search do @doc """ Parses the keyword from the given file. - Returns the stream for each line in the csv file as {ok: result}. + Returns the stream for each line in the csv file as [line_result]. Raise an exception if the file content type is not supported or the file parsing is failed. ### Examples diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index 9886309..02172d8 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -13,7 +13,7 @@ defmodule GoogleCrawlerWeb.UploadController do result = Search.parse_keywords_from_file!(file.path, file.content_type) # TODO: Save these keywords and triggers the task to google search for each keyword - text(conn, result |> Enum.map(fn {:ok, [keyword]} -> keyword end) |> Enum.join(", ")) + text(conn, result |> Enum.map(fn keyword -> List.first(keyword) end) |> Enum.join(", ")) else conn |> put_flash(:error, gettext("Invalid file, please select again.")) diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex index 4d54c70..d357be9 100644 --- a/lib/google_crawler_web/templates/keyword/_form.html.eex +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -3,7 +3,7 @@

<%= gettext("📝 Please put one keyword per line") %>

<%= form_for @changeset, Routes.upload_path(@conn, :create), [multipart: true], fn f -> %> <%= label f, :file %> - <%= file_input f, :file, required: true, accept: "text/csv" %> + <%= file_input f, :file, required: true %> <%= error_tag f, :file %> <%= submit gettext("Upload") %> diff --git a/test/fixtures/keyword_files/invalid_keyword.csv b/test/fixtures/keyword_files/invalid_keyword.csv index e7b0d58..b9f1173 100644 --- a/test/fixtures/keyword_files/invalid_keyword.csv +++ b/test/fixtures/keyword_files/invalid_keyword.csv @@ -1 +1,2 @@ -elixir ruby javascript +elixir, ruby +javascript diff --git a/test/google_crawler_web/controllers/registration_controller_test.exs b/test/google_crawler_web/controllers/registration_controller_test.exs index e08e789..8de7cbb 100644 --- a/test/google_crawler_web/controllers/registration_controller_test.exs +++ b/test/google_crawler_web/controllers/registration_controller_test.exs @@ -19,22 +19,20 @@ defmodule GoogleCrawlerWeb.RegistrationControllerTest do assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end - describe "create/2" do - test "redirects to page index when the data is valid", %{conn: conn} do - user_attrs = UserFactory.build_attrs() + test "create/2 redirects to page index when the data is valid", %{conn: conn} do + user_attrs = UserFactory.build_attrs() - conn = post(conn, Routes.registration_path(conn, :create, user: user_attrs)) + conn = post(conn, Routes.registration_path(conn, :create, user: user_attrs)) - assert redirected_to(conn) == Routes.dashboard_path(conn, :index) - assert get_flash(conn, :info) == "You have signed up successfully!" - end + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) + assert get_flash(conn, :info) == "You have signed up successfully!" + end - test "renders the error when the data is invalid", %{conn: conn} do - user_attrs = UserFactory.build_attrs(%{email: nil, username: nil, password: nil}) + test "create/2 renders the error when the data is invalid", %{conn: conn} do + user_attrs = UserFactory.build_attrs(%{email: nil, username: nil, password: nil}) - conn = post(conn, Routes.registration_path(conn, :create, user: user_attrs)) + conn = post(conn, Routes.registration_path(conn, :create, user: user_attrs)) - assert html_response(conn, 200) =~ "Create Account" - end + assert html_response(conn, 200) =~ "Create Account" end end diff --git a/test/google_crawler_web/controllers/session_controller_test.exs b/test/google_crawler_web/controllers/session_controller_test.exs index d553aa6..c0a6958 100644 --- a/test/google_crawler_web/controllers/session_controller_test.exs +++ b/test/google_crawler_web/controllers/session_controller_test.exs @@ -9,7 +9,7 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do assert html_response(conn, 200) =~ "Sign in" end - test "new/2 redirects to the page controller if the user has already logged in", %{conn: conn} do + test "new/2 redirects to the user dashboard if the user has already logged in", %{conn: conn} do user = UserFactory.create() conn = @@ -19,7 +19,7 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end - test "create/2 redirects to page controller when user credentials are valid", %{conn: conn} do + test "create/2 redirects to user dashboard when user credentials are valid", %{conn: conn} do user = UserFactory.create() conn = @@ -44,7 +44,7 @@ defmodule GoogleCrawlerWeb.SessionControllerTest do assert get_flash(conn, :error) == "The email or password is incorrect, please try again" end - test "delete/2 clears the user session and redirects to the log in page", %{conn: conn} do + test "delete/2 clears the user session and redirects to the login page", %{conn: conn} do user = UserFactory.create() conn = diff --git a/test/google_crawler_web/controllers/upload_controller_test.exs b/test/google_crawler_web/controllers/upload_controller_test.exs new file mode 100644 index 0000000..4196b8a --- /dev/null +++ b/test/google_crawler_web/controllers/upload_controller_test.exs @@ -0,0 +1,50 @@ +defmodule GoogleCrawlerWeb.UploadControllerTest do + use GoogleCrawlerWeb.ConnCase + + alias GoogleCrawler.UserFactory + + test "create/2 renders csv content as text if the keyword file is valid", %{conn: conn} do + user = UserFactory.create() + upload_file = upload_file_fixture("keyword_files/valid_keyword.csv") + + conn = + build_authenticated_conn(user) + |> post(Routes.upload_path(conn, :create), %{keyword_file: %{file: upload_file}}) + + assert text_response(conn, 200) == "elixir, ruby, javascript" + end + + test "create/2 raises error if the file is failed to parse" do + user = UserFactory.create() + upload_file = upload_file_fixture("keyword_files/invalid_keyword.csv") + + conn = build_authenticated_conn(user) + + assert_raise CSV.RowLengthError, fn -> + post(conn, Routes.upload_path(conn, :create), %{keyword_file: %{file: upload_file}}) + end + end + + test "create/2 redirects to the user dashboard with an error message if the file is not supported", + %{conn: conn} do + user = UserFactory.create() + upload_file = upload_file_fixture("keyword_files/unsupported_keyword.txt") + + conn = + build_authenticated_conn(user) + |> post(Routes.upload_path(conn, :create), %{keyword_file: %{file: upload_file}}) + + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) + assert get_flash(conn, :error) == "Invalid file, please select again." + end + + defp upload_file_fixture(path) do + path = Path.join(["test/fixtures", path]) + + %Plug.Upload{ + content_type: MIME.from_path(path), + filename: Path.basename(path), + path: path + } + end +end From dc80c703c9a4cd36e7432daeec9730ed483b3f9f Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 14:36:09 +0700 Subject: [PATCH 08/11] Add context test --- lib/google_crawler/errors.ex | 3 +++ lib/google_crawler/search/keyword_file.ex | 3 ++- test/google_crawler/search_test.exs | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 lib/google_crawler/errors.ex diff --git a/lib/google_crawler/errors.ex b/lib/google_crawler/errors.ex new file mode 100644 index 0000000..0521f3a --- /dev/null +++ b/lib/google_crawler/errors.ex @@ -0,0 +1,3 @@ +defmodule GoogleCrawler.Errors.FileNotSupportedError do + defexception message: "File is not supported" +end diff --git a/lib/google_crawler/search/keyword_file.ex b/lib/google_crawler/search/keyword_file.ex index 1bbf5f8..8c4376f 100644 --- a/lib/google_crawler/search/keyword_file.ex +++ b/lib/google_crawler/search/keyword_file.ex @@ -22,7 +22,8 @@ defmodule GoogleCrawler.Search.KeywordFile do end def parse!(_file_path, _unexpected_content_type) do - raise "File with this extenstion is not supported" + raise GoogleCrawler.Errors.FileNotSupportedError, + message: "File with this extenstion is not supported" end defp validate_file_ext(changeset) do diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 06f6154..9aef9c8 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -37,4 +37,25 @@ defmodule GoogleCrawler.SearchTest do assert {:error, %Ecto.Changeset{}} = Search.create_keyword(@invalid_attrs) end end + + describe "keyword file" do + test "parse_keywords_from_file!/2 returns the stream" do + csv_stream = + Search.parse_keywords_from_file!( + "test/fixtures/keyword_files/valid_keyword.csv", + "text/csv" + ) + + assert Enum.to_list(csv_stream) == [["elixir"], ["ruby"], ["javascript"]] + end + + test "parse_keywords_from_file!/2 raises an exception when the file type is not supported" do + assert_raise GoogleCrawler.Errors.FileNotSupportedError, fn -> + Search.parse_keywords_from_file!( + "test/fixtures/keyword_files/unsupported_keyword.txt", + "text/plain" + ) + end + end + end end From 41abc082d98acb47355001dcb408898c18a653aa Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 14:40:48 +0700 Subject: [PATCH 09/11] Fix typo and rename arguments --- lib/google_crawler/search.ex | 6 +++--- lib/google_crawler/search/keyword_file.ex | 4 ++-- lib/google_crawler_web/templates/keyword/_form.html.eex | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 6c3b7bc..913aaf4 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -59,7 +59,7 @@ defmodule GoogleCrawler.Search do @doc """ Parses the keyword from the given file. Returns the stream for each line in the csv file as [line_result]. - Raise an exception if the file content type is not supported or the file parsing is failed. + Raise an exception if the file mime type is not supported or the file parsing is failed. ### Examples @@ -67,7 +67,7 @@ defmodule GoogleCrawler.Search do [ok: ["hotels"], ok: ["restaurants"]] """ - def parse_keywords_from_file!(file_path, content_type) do - KeywordFile.parse!(file_path, content_type) + def parse_keywords_from_file!(file_path, mime_type) do + KeywordFile.parse!(file_path, mime_type) end end diff --git a/lib/google_crawler/search/keyword_file.ex b/lib/google_crawler/search/keyword_file.ex index 8c4376f..fe3e60a 100644 --- a/lib/google_crawler/search/keyword_file.ex +++ b/lib/google_crawler/search/keyword_file.ex @@ -21,9 +21,9 @@ defmodule GoogleCrawler.Search.KeywordFile do |> CSV.decode!() end - def parse!(_file_path, _unexpected_content_type) do + def parse!(_file_path, _unexpected_mime_type) do raise GoogleCrawler.Errors.FileNotSupportedError, - message: "File with this extenstion is not supported" + message: "File with this extension is not supported" end defp validate_file_ext(changeset) do diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex index d357be9..d77ceae 100644 --- a/lib/google_crawler_web/templates/keyword/_form.html.eex +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -14,4 +14,3 @@

<%= gettext("Keywords") %>

<%= gettext("You don't have any keywords.") %>

- From b57fd097e181ee5852f5bb9f57f4fba931867ed8 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 09:56:09 +0700 Subject: [PATCH 10/11] Add factory for keyword and split the list from form view --- .../templates/dashboard/index.html.eex | 5 +++ .../templates/keyword/_form.html.eex | 5 --- test/factories/keyword_factory.ex | 21 +++++++++++ test/google_crawler/accounts_test.exs | 3 +- test/google_crawler/search_test.exs | 36 ++++++++----------- 5 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 test/factories/keyword_factory.ex diff --git a/lib/google_crawler_web/templates/dashboard/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex index 385bc64..a3bf664 100644 --- a/lib/google_crawler_web/templates/dashboard/index.html.eex +++ b/lib/google_crawler_web/templates/dashboard/index.html.eex @@ -1 +1,6 @@ <%= render GoogleCrawlerWeb.KeywordView, "_form.html", assigns %> +
+
+

<%= gettext("Keywords") %>

+

<%= gettext("You don't have any keywords.") %>

+
diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex index d77ceae..d0f3471 100644 --- a/lib/google_crawler_web/templates/keyword/_form.html.eex +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -9,8 +9,3 @@ <%= submit gettext("Upload") %> <% end %> -
-
-

<%= gettext("Keywords") %>

-

<%= gettext("You don't have any keywords.") %>

-
diff --git a/test/factories/keyword_factory.ex b/test/factories/keyword_factory.ex new file mode 100644 index 0000000..597c333 --- /dev/null +++ b/test/factories/keyword_factory.ex @@ -0,0 +1,21 @@ +defmodule GoogleCrawler.KeywordFactory do + alias GoogleCrawler.Search + + def default_attrs do + %{ + keyword: FakerElixir.Lorem.word() + } + end + + def build_attrs(attrs \\ %{}) do + Enum.into(attrs, default_attrs()) + end + + def create(attrs \\ %{}) do + keyword_attrs = build_attrs(attrs) + + {:ok, keyword} = Search.create_keyword(keyword_attrs) + + keyword + end +end diff --git a/test/google_crawler/accounts_test.exs b/test/google_crawler/accounts_test.exs index 566a67c..f7dbaf6 100644 --- a/test/google_crawler/accounts_test.exs +++ b/test/google_crawler/accounts_test.exs @@ -2,11 +2,10 @@ defmodule GoogleCrawler.AccountsTest do use GoogleCrawler.DataCase alias GoogleCrawler.Accounts + alias GoogleCrawler.Accounts.User alias GoogleCrawler.UserFactory describe "users" do - alias GoogleCrawler.Accounts.User - test "get_user/1 returns the user with given id" do user = UserFactory.create() diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 9aef9c8..ceb3df5 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -2,39 +2,33 @@ defmodule GoogleCrawler.SearchTest do use GoogleCrawler.DataCase alias GoogleCrawler.Search + alias GoogleCrawler.Search.Keyword + alias GoogleCrawler.KeywordFactory describe "keywords" do - alias GoogleCrawler.Search.Keyword - - @valid_attrs %{keyword: "some keyword"} - @invalid_attrs %{keyword: nil} - - def keyword_fixture(attrs \\ %{}) do - {:ok, keyword} = - attrs - |> Enum.into(@valid_attrs) - |> Search.create_keyword() - - keyword - end - test "list_keywords/0 returns all keywords" do - keyword = keyword_fixture() - assert Search.list_keywords() == [keyword] + keyword = KeywordFactory.create() + + assert Search.list_keywords() |> Enum.map(&Map.get(&1, :keyword)) == [keyword.keyword] end test "get_keyword/1 returns the keyword with given id" do - keyword = keyword_fixture() - assert Search.get_keyword(keyword.id) == keyword + keyword = KeywordFactory.create() + + assert Search.get_keyword(keyword.id).keyword == keyword.keyword end test "create_keyword/1 with valid data creates a keyword" do - assert {:ok, %Keyword{} = keyword} = Search.create_keyword(@valid_attrs) - assert keyword.keyword == "some keyword" + keyword_attrs = KeywordFactory.build_attrs(%{keyword: "elixir"}) + + assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs) + assert keyword.keyword == "elixir" end test "create_keyword/1 with invalid data returns error changeset" do - assert {:error, %Ecto.Changeset{}} = Search.create_keyword(@invalid_attrs) + keyword_attrs = KeywordFactory.build_attrs(%{keyword: ""}) + + assert {:error, %Ecto.Changeset{}} = Search.create_keyword(keyword_attrs) end end From 266d7257cf7976cbac754152373cb48dfa514af9 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Fri, 17 Apr 2020 16:21:15 +0700 Subject: [PATCH 11/11] Update the doc and the tests for keyword file --- lib/google_crawler/search.ex | 6 ++-- test/factories/keyword_file_factory.ex | 13 ++++++++ .../search/keyword_file_test.exs | 30 +++++++++++++++++++ .../controllers/upload_controller_test.exs | 12 ++------ test/support/fixture_helper.ex | 13 ++++++++ 5 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 test/factories/keyword_file_factory.ex create mode 100644 test/google_crawler/search/keyword_file_test.exs create mode 100644 test/support/fixture_helper.ex diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 913aaf4..db39dac 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -29,11 +29,11 @@ defmodule GoogleCrawler.Search do ## Examples - iex> get_keyword!(123) + iex> get_keyword(123) %Keyword{} - iex> get_keyword!(456) - ** (Ecto.NoResultsError) + iex> get_keyword(456) + nil """ def get_keyword(id), do: Repo.get(Keyword, id) diff --git a/test/factories/keyword_file_factory.ex b/test/factories/keyword_file_factory.ex new file mode 100644 index 0000000..6000058 --- /dev/null +++ b/test/factories/keyword_file_factory.ex @@ -0,0 +1,13 @@ +defmodule GoogleCrawler.KeywordFileFactory do + import GoogleCrawler.FixtureHelper + + def default_attrs do + %{ + file: upload_file_fixture("keyword_files/invalid_keyword.csv") + } + end + + def build_attrs(attrs \\ %{}) do + Enum.into(attrs, default_attrs()) + end +end diff --git a/test/google_crawler/search/keyword_file_test.exs b/test/google_crawler/search/keyword_file_test.exs new file mode 100644 index 0000000..0daf16d --- /dev/null +++ b/test/google_crawler/search/keyword_file_test.exs @@ -0,0 +1,30 @@ +defmodule GoogleCrawler.KeywordFileTest do + use GoogleCrawler.DataCase + + import GoogleCrawler.FixtureHelper + + alias GoogleCrawler.KeywordFileFactory + alias GoogleCrawler.Search.KeywordFile + + describe "changeset" do + test "file is required" do + attrs = KeywordFileFactory.build_attrs(%{file: nil}) + changeset = KeywordFile.changeset(%KeywordFile{}, attrs) + + refute changeset.valid? + assert %{file: ["can't be blank"]} = errors_on(changeset) + end + + test "file is in the allowed extensions" do + attrs = + KeywordFileFactory.build_attrs(%{ + file: upload_file_fixture("keyword_files/unsupported_keyword.txt") + }) + + changeset = KeywordFile.changeset(%KeywordFile{}, attrs) + + refute changeset.valid? + assert %{file: ["is not supported"]} = errors_on(changeset) + end + end +end diff --git a/test/google_crawler_web/controllers/upload_controller_test.exs b/test/google_crawler_web/controllers/upload_controller_test.exs index 4196b8a..b95f375 100644 --- a/test/google_crawler_web/controllers/upload_controller_test.exs +++ b/test/google_crawler_web/controllers/upload_controller_test.exs @@ -1,6 +1,8 @@ defmodule GoogleCrawlerWeb.UploadControllerTest do use GoogleCrawlerWeb.ConnCase + import GoogleCrawler.FixtureHelper + alias GoogleCrawler.UserFactory test "create/2 renders csv content as text if the keyword file is valid", %{conn: conn} do @@ -37,14 +39,4 @@ defmodule GoogleCrawlerWeb.UploadControllerTest do assert redirected_to(conn) == Routes.dashboard_path(conn, :index) assert get_flash(conn, :error) == "Invalid file, please select again." end - - defp upload_file_fixture(path) do - path = Path.join(["test/fixtures", path]) - - %Plug.Upload{ - content_type: MIME.from_path(path), - filename: Path.basename(path), - path: path - } - end end diff --git a/test/support/fixture_helper.ex b/test/support/fixture_helper.ex new file mode 100644 index 0000000..310bc94 --- /dev/null +++ b/test/support/fixture_helper.ex @@ -0,0 +1,13 @@ +defmodule GoogleCrawler.FixtureHelper do + @fixture_path "test/fixtures" + + def upload_file_fixture(path) do + path = Path.join([@fixture_path, path]) + + %Plug.Upload{ + content_type: MIME.from_path(path), + filename: Path.basename(path), + path: path + } + end +end