diff --git a/lib/something_erlang/awful_api/awful_api.ex b/lib/something_erlang/awful_api/awful_api.ex index e70cb9d..070675e 100644 --- a/lib/something_erlang/awful_api/awful_api.ex +++ b/lib/something_erlang/awful_api/awful_api.ex @@ -1,8 +1,7 @@ defmodule SomethingErlang.AwfulApi do require Logger - alias SomethingErlang.AwfulApi.Thread - alias SomethingErlang.AwfulApi.Bookmarks + alias SomethingErlang.AwfulApi.{Client, Thread, Bookmarks} @doc """ Returns a list of all posts on page of a thread. @@ -16,10 +15,12 @@ defmodule SomethingErlang.AwfulApi do 12 """ def parsed_thread(id, page, user) do - Thread.compile(id, page, user) + Client.thread_doc(id, page, user) + |> Thread.compile() end - def bookmarks(user) do - Bookmarks.compile(1, user) + def bookmarks(page, user) do + Client.bookmarks_doc(page, user) + |> Bookmarks.compile() end end diff --git a/lib/something_erlang/awful_api/bookmarks.ex b/lib/something_erlang/awful_api/bookmarks.ex index d0097bd..b318378 100644 --- a/lib/something_erlang/awful_api/bookmarks.ex +++ b/lib/something_erlang/awful_api/bookmarks.ex @@ -1,59 +1,80 @@ defmodule SomethingErlang.AwfulApi.Bookmarks do + import Meeseeks.CSS + require Logger - alias SomethingErlang.AwfulApi.Client + def compile(html) do + for thread <- Meeseeks.all(html, css("tr.thread")) do + thread_id = + Meeseeks.attr(thread, "id") + |> String.split("thread") + |> List.last() + |> String.to_integer() - def compile(page, user) do - doc = Client.bookmarks_doc(page, user) - html = Floki.parse_document!(doc) - - for thread <- Floki.find(html, "tr.thread") do parse(thread) + |> Map.put(:id, thread_id) end end def parse(thread) do - %{ - title: Floki.find(thread, "td.title") |> inner_html() |> Floki.raw_html(), - icon: Floki.find(thread, "td.icon") |> inner_html() |> Floki.raw_html(), - author: Floki.find(thread, "td.author") |> inner_html() |> Floki.text(), - replies: Floki.find(thread, "td.replies") |> inner_html() |> Floki.text(), - views: Floki.find(thread, "td.views") |> inner_html() |> Floki.text(), - rating: Floki.find(thread, "td.rating") |> inner_html() |> Floki.raw_html(), - lastpost: Floki.find(thread, "td.lastpost") |> inner_html() |> Floki.raw_html() - } - - for {"td", [{"class", class} | _attrs], children} <- Floki.find(thread, "td"), - String.starts_with?(class, "star") == false, + for col <- Meeseeks.all(thread, css("td:not(.star)")), + class = Meeseeks.attr(col, "class") |> String.split() |> List.first(), into: %{} do - case class do - <<"title", _rest::binary>> -> - {:title, children |> Floki.raw_html()} - - <<"icon", _rest::binary>> -> - {:icon, children |> Floki.raw_html()} - - <<"author", _rest::binary>> -> - {:author, children |> Floki.text()} - - <<"replies", _rest::binary>> -> - {:replies, children |> Floki.text() |> String.to_integer()} - - <<"views", _rest::binary>> -> - {:views, children |> Floki.text() |> String.to_integer()} - - <<"rating", _rest::binary>> -> - {:rating, children |> Floki.raw_html()} - - <<"lastpost", _rest::binary>> -> - {:lastpost, children |> Floki.raw_html()} - end + {String.to_atom(class), thread_data(class, col)} end end - defp inner_html(node) do - node - |> List.first() - |> Floki.children() + defp thread_data("icon", td) do + img = Meeseeks.one(td, css("img")) + + %{ + icon: Meeseeks.attr(img, "src"), + title: Meeseeks.attr(img, "alt") + } end + + defp thread_data("title", td) do + last_seen = Meeseeks.one(td, css(".lastseen .count")) + info = Meeseeks.one(td, css(".info")) + + %{ + new_posts: Meeseeks.text(last_seen) |> String.to_integer(), + thread_title: Meeseeks.text(Meeseeks.one(info, css(".thread_title"))) + } + end + + defp thread_data("author", td) do + Meeseeks.text(td) + end + + defp thread_data("replies", td) do + Meeseeks.text(td) + |> String.to_integer() + end + + defp thread_data("views", td) do + Meeseeks.text(td) + |> String.to_integer() + end + + defp thread_data("rating", td) do + img = Meeseeks.one(td, css("img")) + + %{ + icon: Meeseeks.attr(img, "src"), + title: Meeseeks.attr(img, "title") + } + end + + defp thread_data("lastpost", td) do + date = Meeseeks.one(td, css(".date")) + author = Meeseeks.one(td, css(".author")) + + %{ + date: Meeseeks.text(date), + author: Meeseeks.text(author) + } + end + + defp thread_data(class, _td), do: Logger.error("#{inspect(class)} not found") end diff --git a/lib/something_erlang/awful_api/thread.ex b/lib/something_erlang/awful_api/thread.ex index 8d00102..93ff4d1 100644 --- a/lib/something_erlang/awful_api/thread.ex +++ b/lib/something_erlang/awful_api/thread.ex @@ -1,39 +1,60 @@ defmodule SomethingErlang.AwfulApi.Thread do + import Meeseeks.CSS + require Logger - alias SomethingErlang.AwfulApi.Client + def compile(html) do + title = + Meeseeks.one(html, css("title")) + |> Meeseeks.text() + |> String.replace(" - The Something Awful Forums", "") - def compile(id, page, user) do - doc = Client.thread_doc(id, page, user) - html = Floki.parse_document!(doc) - thread = Floki.find(html, "#thread") |> Floki.filter_out("table.post.ignored") + thread = + Meeseeks.one(html, css("#thread")) - title = Floki.find(html, "title") |> Floki.text() - title = title |> String.replace(" - The Something Awful Forums", "") + thread_id = + Meeseeks.attr(thread, "class") + |> String.split(":") + |> List.last() + |> String.to_integer() + + page = + Meeseeks.one(html, css("#content .pages.top option[selected]")) + |> Meeseeks.text() + |> case do + "" -> 1 + s -> String.to_integer(s) + end page_count = - case Floki.find(html, "#content .pages.top option:last-of-type") |> Floki.text() do + Meeseeks.one(html, css("#content .pages.top option:last-of-type")) + |> Meeseeks.text() + |> case do "" -> 1 s -> String.to_integer(s) end posts = - for post <- Floki.find(thread, "table.post") do + for post <- Meeseeks.all(thread, css("table.post:not(.ignored)")) do %{ - userinfo: post |> userinfo(), - postdate: post |> postdate(), - postbody: post |> postbody() + userinfo: userinfo(post), + postdate: postdate(post), + postbody: postbody(post) } end - %{id: id, title: title, page: page, page_count: page_count, posts: posts} + %{id: thread_id, title: title, page: page, page_count: page_count, posts: posts} end defp userinfo(post) do - user = Floki.find(post, "dl.userinfo") - name = user |> Floki.find("dt") |> Floki.text() - regdate = user |> Floki.find("dd.registered") |> Floki.text() - title = user |> Floki.find("dd.title") |> List.first() |> Floki.children() |> Floki.raw_html() + user = Meeseeks.one(post, css("dl.userinfo")) + name = user |> Meeseeks.one(css("dt")) |> Meeseeks.text() + regdate = user |> Meeseeks.one(css("dd.registered")) |> Meeseeks.text() + + title = + user + |> Meeseeks.one(css("dd.title > *")) + |> Meeseeks.html() %{ name: name, @@ -43,12 +64,15 @@ defmodule SomethingErlang.AwfulApi.Thread do end defp postdate(post) do - date = Floki.find(post, "td.postdate") |> Floki.find("td.postdate") |> Floki.text() + date = + post + |> Meeseeks.one(css("td.postdate")) + |> Meeseeks.text() [month_text, day, year, hours, minutes] = date |> String.split(~r{[\s,:]}, trim: true) - |> Enum.drop(1) + |> Enum.drop(2) month = 1 + @@ -68,23 +92,20 @@ defmodule SomethingErlang.AwfulApi.Thread do end defp postbody(post) do - body = - Floki.find(post, "td.postbody") - |> List.first() - |> Floki.filter_out(:comment) + {_, _, body} = + post + |> Meeseeks.one(css("td.postbody")) + |> Meeseeks.tree() - Floki.traverse_and_update(body, fn - {"img", attrs, []} -> transform(:img, attrs) - {"a", attrs, children} -> transform(:a, attrs, children) - other -> other - end) - |> Floki.children() - |> Floki.raw_html() + body + |> Enum.map(&transform/1) + |> Enum.reject(fn x -> x == "" end) + |> then(&{"div", [], &1}) + |> Meeseeks.parse(:tuple_tree) + |> Meeseeks.html() end - defp transform(elem, attr, children \\ []) - - defp transform(:img, attrs, _children) do + defp transform({"img", attrs, _children}) do {"class", class} = List.keyfind(attrs, "class", 0, {"class", ""}) if class == "sa-smilie" do @@ -95,7 +116,7 @@ defmodule SomethingErlang.AwfulApi.Thread do end end - defp transform(:a, attrs, children) do + defp transform({"a", attrs, children}) do {"href", href} = List.keyfind(attrs, "href", 0, {"href", ""}) cond do @@ -124,6 +145,12 @@ defmodule SomethingErlang.AwfulApi.Thread do end end + defp transform({:comment, _}), do: "" + defp transform({tag, attrs, children}), do: {tag, attrs, children} + + defp transform(text) when is_binary(text), + do: String.trim(text) + defp transform_link(:mp4, href), do: {"div", [{"class", "responsive-embed"}], diff --git a/lib/something_erlang/grover.ex b/lib/something_erlang/grover.ex index 8a907a6..5ad8cd4 100644 --- a/lib/something_erlang/grover.ex +++ b/lib/something_erlang/grover.ex @@ -55,8 +55,8 @@ defmodule SomethingErlang.Grover do end @impl true - def handle_call({:show_bookmarks, _page_number}, _from, state) do - bookmarks = AwfulApi.bookmarks(state.user) + def handle_call({:show_bookmarks, page_number}, _from, state) do + bookmarks = AwfulApi.bookmarks(page_number, state.user) {:reply, bookmarks, state} end diff --git a/lib/something_erlang_web/live/bookmarks_live.ex b/lib/something_erlang_web/live/bookmarks_live.ex new file mode 100644 index 0000000..a962c33 --- /dev/null +++ b/lib/something_erlang_web/live/bookmarks_live.ex @@ -0,0 +1,42 @@ +defmodule SomethingErlangWeb.BookmarksLive do + use SomethingErlangWeb, :live_view + + alias SomethingErlang.Grover + + def render(assigns) do + ~H""" +