rdiedrich/something-erlang
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new base

Browse Source
This commit is contained in:
Rüdiger Diedrich
2024-06-02 13:31:19 +02:00
parent 9d547bcdf2
commit 443081e086
67 changed files with 623 additions and 4282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
lib/something_erlang/accounts.ex
View File

@ -1,114 +0,0 @@
defmodule SomethingErlang.Accounts do
@moduledoc """
The Accounts context.
"""
import Ecto.Query, warn: false
alias SomethingErlang.Repo
alias SomethingErlang.AwfulApi.Client
alias SomethingErlang.Accounts.{User, UserToken}
## Database getters
def get_user_by_bbuserid(bbuserid) when is_binary(bbuserid) do
Repo.get_by(User, :bbuserid, bbuserid)
end
def get_or_create_user_by_bbuserid(bbuserid)
when is_binary(bbuserid) do
if user = Repo.get_by(User, bbuserid: bbuserid) do
user
else
%User{bbuserid: bbuserid} |> Repo.insert!()
end
end
def login_sa_user_and_get_cookies(username, password)
when is_binary(username) and is_binary(password) do
case Client.login(username, password) do
%{bbuserid: userid, bbpassword: _hash} = bbuser ->
user = get_or_create_user_by_bbuserid(userid)
Map.merge(user, bbuser)
_ ->
nil
end
end
@doc """
Gets a single user.
Raises `Ecto.NoResultsError` if the User does not exist.
## Examples
iex> get_user!(123)
%User{}
iex> get_user!(456)
** (Ecto.NoResultsError)
"""
def get_user!(id), do: Repo.get!(User, id)
## User registration
@doc """
Registers a user.
## Examples
iex> register_user(%{field: value})
{:ok, %User{}}
iex> register_user(%{field: bad_value})
{:error, %Ecto.Changeset{}}
"""
def register_user(attrs) do
%User{}
|> User.registration_changeset(attrs)
|> Repo.insert()
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking user changes.
## Examples
iex> change_user_registration(user)
%Ecto.Changeset{data: %User{}}
"""
def change_user_registration(%User{} = user, attrs \\ %{}) do
User.registration_changeset(user, attrs)
end
## Session
@doc """
Generates a session token.
"""
def generate_user_session_token(user) do
{token, user_token} = UserToken.build_session_token(user)
Repo.insert!(user_token)
token
end
@doc """
Gets the user with the given signed token.
"""
def get_user_by_session_token(token) do
{:ok, query} = UserToken.verify_session_token_query(token)
Repo.one(query)
end
@doc """
Deletes the signed token with the given context.
"""
def delete_user_session_token(token) do
Repo.delete_all(UserToken.token_and_context_query(token, "session"))
:ok
end
end

16
lib/something_erlang/accounts/user.ex
View File

@ -1,16 +0,0 @@
defmodule SomethingErlang.Accounts.User do
use Ecto.Schema
import Ecto.Changeset
schema "users" do
field :bbuserid, :string
timestamps()
end
def registration_changeset(user, attrs, _opts \\ []) do
user
|> cast(attrs, [:bbuserid])
|> validate_required([:bbuserid])
end
end

79
lib/something_erlang/accounts/user_notifier.ex
View File

@ -1,79 +0,0 @@
defmodule SomethingErlang.Accounts.UserNotifier do
import Swoosh.Email
alias SomethingErlang.Mailer
# Delivers the email using the application mailer.
defp deliver(recipient, subject, body) do
email =
new()
|> to(recipient)
|> from({"SomethingErlang", "contact@example.com"})
|> subject(subject)
|> text_body(body)
with {:ok, _metadata} <- Mailer.deliver(email) do
{:ok, email}
end
end
@doc """
Deliver instructions to confirm account.
"""
def deliver_confirmation_instructions(user, url) do
deliver(user.email, "Confirmation instructions", """
==============================
Hi #{user.email},
You can confirm your account by visiting the URL below:
#{url}
If you didn't create an account with us, please ignore this.
==============================
""")
end
@doc """
Deliver instructions to reset a user password.
"""
def deliver_reset_password_instructions(user, url) do
deliver(user.email, "Reset password instructions", """
==============================
Hi #{user.email},
You can reset your password by visiting the URL below:
#{url}
If you didn't request this change, please ignore this.
==============================
""")
end
@doc """
Deliver instructions to update a user email.
"""
def deliver_update_email_instructions(user, url) do
deliver(user.email, "Update email instructions", """
==============================
Hi #{user.email},
You can change your email by visiting the URL below:
#{url}
If you didn't request this change, please ignore this.
==============================
""")
end
end

179
lib/something_erlang/accounts/user_token.ex
View File

@ -1,179 +0,0 @@
defmodule SomethingErlang.Accounts.UserToken do
use Ecto.Schema
import Ecto.Query
alias SomethingErlang.Accounts.UserToken
@hash_algorithm :sha256
@rand_size 32
# It is very important to keep the reset password token expiry short,
# since someone with access to the email may take over the account.
@reset_password_validity_in_days 1
@confirm_validity_in_days 7
@change_email_validity_in_days 7
@session_validity_in_days 60
schema "users_tokens" do
field :token, :binary
field :context, :string
field :sent_to, :string
belongs_to :user, SomethingErlang.Accounts.User
timestamps(updated_at: false)
end
@doc """
Generates a token that will be stored in a signed place,
such as session or cookie. As they are signed, those
tokens do not need to be hashed.
The reason why we store session tokens in the database, even
though Phoenix already provides a session cookie, is because
Phoenix' default session cookies are not persisted, they are
simply signed and potentially encrypted. This means they are
valid indefinitely, unless you change the signing/encryption
salt.
Therefore, storing them allows individual user
sessions to be expired. The token system can also be extended
to store additional data, such as the device used for logging in.
You could then use this information to display all valid sessions
and devices in the UI and allow users to explicitly expire any
session they deem invalid.
"""
def build_session_token(user) do
token = :crypto.strong_rand_bytes(@rand_size)
{token, %UserToken{token: token, context: "session", user_id: user.id}}
end
@doc """
Checks if the token is valid and returns its underlying lookup query.
The query returns the user found by the token, if any.
The token is valid if it matches the value in the database and it has
not expired (after @session_validity_in_days).
"""
def verify_session_token_query(token) do
query =
from token in token_and_context_query(token, "session"),
join: user in assoc(token, :user),
where: token.inserted_at > ago(@session_validity_in_days, "day"),
select: user
{:ok, query}
end
@doc """
Builds a token and its hash to be delivered to the user's email.
The non-hashed token is sent to the user email while the
hashed part is stored in the database. The original token cannot be reconstructed,
which means anyone with read-only access to the database cannot directly use
the token in the application to gain access. Furthermore, if the user changes
their email in the system, the tokens sent to the previous email are no longer
valid.
Users can easily adapt the existing code to provide other types of delivery methods,
for example, by phone numbers.
"""
def build_email_token(user, context) do
build_hashed_token(user, context, user.email)
end
defp build_hashed_token(user, context, sent_to) do
token = :crypto.strong_rand_bytes(@rand_size)
hashed_token = :crypto.hash(@hash_algorithm, token)
{Base.url_encode64(token, padding: false),
%UserToken{
token: hashed_token,
context: context,
sent_to: sent_to,
user_id: user.id
}}
end
@doc """
Checks if the token is valid and returns its underlying lookup query.
The query returns the user found by the token, if any.
The given token is valid if it matches its hashed counterpart in the
database and the user email has not changed. This function also checks
if the token is being used within a certain period, depending on the
context. The default contexts supported by this function are either
"confirm", for account confirmation emails, and "reset_password",
for resetting the password. For verifying requests to change the email,
see `verify_change_email_token_query/2`.
"""
def verify_email_token_query(token, context) do
case Base.url_decode64(token, padding: false) do
{:ok, decoded_token} ->
hashed_token = :crypto.hash(@hash_algorithm, decoded_token)
days = days_for_context(context)
query =
from token in token_and_context_query(hashed_token, context),
join: user in assoc(token, :user),
where: token.inserted_at > ago(^days, "day") and token.sent_to == user.email,
select: user
{:ok, query}
:error ->
:error
end
end
defp days_for_context("confirm"), do: @confirm_validity_in_days
defp days_for_context("reset_password"), do: @reset_password_validity_in_days
@doc """
Checks if the token is valid and returns its underlying lookup query.
The query returns the user found by the token, if any.
This is used to validate requests to change the user
email. It is different from `verify_email_token_query/2` precisely because
`verify_email_token_query/2` validates the email has not changed, which is
the starting point by this function.
The given token is valid if it matches its hashed counterpart in the
database and if it has not expired (after @change_email_validity_in_days).
The context must always start with "change:".
"""
def verify_change_email_token_query(token, "change:" <> _ = context) do
case Base.url_decode64(token, padding: false) do
{:ok, decoded_token} ->
hashed_token = :crypto.hash(@hash_algorithm, decoded_token)
query =
from token in token_and_context_query(hashed_token, context),
where: token.inserted_at > ago(@change_email_validity_in_days, "day")
{:ok, query}
:error ->
:error
end
end
@doc """
Returns the token struct for the given token value and context.
"""
def token_and_context_query(token, context) do
from UserToken, where: [token: ^token, context: ^context]
end
@doc """
Gets all tokens for the given user for the given contexts.
"""
def user_and_contexts_query(user, :all) do
from t in UserToken, where: t.user_id == ^user.id
end
def user_and_contexts_query(user, [_ | _] = contexts) do
from t in UserToken, where: t.user_id == ^user.id and t.context in ^contexts
end
end

14
lib/something_erlang/application.ex
View File

@ -8,20 +8,16 @@ defmodule SomethingErlang.Application do
@impl true
def start(_type, _args) do
children = [
{Registry, [name: SomethingErlang.Registry.Grovers, keys: :unique]},
{DynamicSupervisor, [name: SomethingErlang.Supervisor.Grovers, strategy: :one_for_one]},
# Start the Telemetry supervisor
SomethingErlangWeb.Telemetry,
# Start the Ecto repository
SomethingErlang.Repo,
# Start the PubSub system
{DNSCluster, query: Application.get_env(:something_erlang, :dns_cluster_query) || :ignore},
{Phoenix.PubSub, name: SomethingErlang.PubSub},
# Start Finch
# Start the Finch HTTP client for sending emails
{Finch, name: SomethingErlang.Finch},
# Start the Endpoint (http/https)
SomethingErlangWeb.Endpoint
# Start a worker by calling: SomethingErlang.Worker.start_link(arg)
# {SomethingErlang.Worker, arg}
# {SomethingErlang.Worker, arg},
# Start to serve requests, typically the last entry
SomethingErlangWeb.Endpoint
]
# See https://hexdocs.pm/elixir/Supervisor.html

25
lib/something_erlang/awful_api/awful_api.ex
View File

@ -1,25 +0,0 @@
defmodule SomethingErlang.AwfulApi do
require Logger
alias SomethingErlang.AwfulApi.Thread
alias SomethingErlang.AwfulApi.Bookmarks
@doc """
Returns a list of all posts on page of a thread.
## Examples
iex> t = AwfulApi.parsed_thread(3945300, 1)
iex> length(t.posts)
42
iex> t.page_count
12
"""
def parsed_thread(id, page, user) do
Thread.compile(id, page, user)
end
def bookmarks(user) do
Bookmarks.compile(1, user)
end
end

59
lib/something_erlang/awful_api/bookmarks.ex
View File

@ -1,59 +0,0 @@
defmodule SomethingErlang.AwfulApi.Bookmarks do
require Logger
alias SomethingErlang.AwfulApi.Client
def compile(page, user) do
doc = Client.bookmarks_doc(page, user)
html = Floki.parse_document!(doc)
for thread <- Floki.find(html, "tr.thread") do
parse(thread)
end
end
def parse(thread) do
%{
title: Floki.find(thread, "td.title") |> inner_html() |> Floki.raw_html(),
icon: Floki.find(thread, "td.icon") |> inner_html() |> Floki.raw_html(),
author: Floki.find(thread, "td.author") |> inner_html() |> Floki.text(),
replies: Floki.find(thread, "td.replies") |> inner_html() |> Floki.text(),
views: Floki.find(thread, "td.views") |> inner_html() |> Floki.text(),
rating: Floki.find(thread, "td.rating") |> inner_html() |> Floki.raw_html(),
lastpost: Floki.find(thread, "td.lastpost") |> inner_html() |> Floki.raw_html()
}
for {"td", [{"class", class} | _attrs], children} <- Floki.find(thread, "td"),
String.starts_with?(class, "star") == false,
into: %{} do
case class do
<<"title", _rest::binary>> ->
{:title, children |> Floki.raw_html()}
<<"icon", _rest::binary>> ->
{:icon, children |> Floki.raw_html()}
<<"author", _rest::binary>> ->
{:author, children |> Floki.text()}
<<"replies", _rest::binary>> ->
{:replies, children |> Floki.text() |> String.to_integer()}
<<"views", _rest::binary>> ->
{:views, children |> Floki.text() |> String.to_integer()}
<<"rating", _rest::binary>> ->
{:rating, children |> Floki.raw_html()}
<<"lastpost", _rest::binary>> ->
{:lastpost, children |> Floki.raw_html()}
end
end
end
defp inner_html(node) do
node
|> List.first()
|> Floki.children()
end
end

95
lib/something_erlang/awful_api/client.ex
View File

@ -1,95 +0,0 @@
defmodule SomethingErlang.AwfulApi.Client do
@base_url "https://forums.somethingawful.com/"
@user_agent "SomethingErlangClient/0.1"
require Logger
def thread_doc(id, page, user) do
resp = new(user) |> get_thread(id, page)
Logger.debug("Client reply in #{resp.private.time}ms ")
:unicode.characters_to_binary(resp.body, :latin1)
end
def thread_lastseen_page(id, user) do
resp = new(user) |> get_thread_newpost(id)
%{status: 302, headers: headers} = resp
{"location", redir_url} = List.keyfind(headers, "location", 0)
[_, page] = Regex.run(~r/pagenumber=(\d+)/, redir_url)
page |> String.to_integer()
end
def bookmarks_doc(page, user) do
resp = new(user) |> get_bookmarks(page)
:unicode.characters_to_binary(resp.body, :latin1)
end
defp get_thread(req, id, page) do
url = "showthread.php"
params = [threadid: id, pagenumber: page]
Req.get!(req, url: url, params: params)
end
defp get_thread_newpost(req, id) do
url = "showthread.php"
params = [threadid: id, goto: "newpost"]
Req.get!(req, url: url, params: params, follow_redirects: false)
end
defp get_bookmarks(req, page) do
url = "bookmarkthreads.php"
params = [pagenumber: page]
Req.get!(req, url: url, params: params)
end
def login(username, password) do
form = [action: "login", username: username, password: password]
url = "account.php"
new()
|> Req.post!(url: url, form: form)
|> extract_cookies()
end
defp extract_cookies(%Req.Response{} = response) do
cookies = response.headers["set-cookie"]
for cookie <- cookies, String.starts_with?(cookie, "bb"), into: %{} do
cookie
|> String.split(";", parts: 2)
|> List.first()
|> String.split("=")
|> then(fn [k, v] -> {String.to_existing_atom(k), v} end)
end
end
defp new(user) do
Req.new(
base_url: @base_url,
user_agent: @user_agent,
cache: true,
headers: [cookie: [cookies(%{bbuserid: user.id, bbpassword: user.hash})]]
)
|> Req.Request.append_request_steps(
time: fn req -> Req.Request.put_private(req, :time, Time.utc_now()) end
)
|> Req.Request.prepend_response_steps(
time: fn {req, res} ->
start = req.private.time
diff = Time.diff(Time.utc_now(), start, :millisecond)
{req, Req.Response.put_private(res, :time, diff)}
end
)
end
defp new() do
Req.new(
base_url: @base_url,
user_agent: @user_agent,
redirect: false
)
end
defp cookies(args) when is_map(args) do
Enum.map_join(args, "; ", fn {k, v} -> "#{k}=#{v}" end)
end
end

170
lib/something_erlang/awful_api/thread.ex
View File

@ -1,170 +0,0 @@
defmodule SomethingErlang.AwfulApi.Thread do
require Logger
alias SomethingErlang.AwfulApi.Client
def compile(id, page, user) do
doc = Client.thread_doc(id, page, user)
html = Floki.parse_document!(doc)
thread = Floki.find(html, "#thread") |> Floki.filter_out("table.post.ignored")
title = Floki.find(html, "title") |> Floki.text()
title = title |> String.replace(" - The Something Awful Forums", "")
page_count =
case Floki.find(html, "#content .pages.top option:last-of-type") |> Floki.text() do
"" -> 1
s -> String.to_integer(s)
end
posts =
for post <- Floki.find(thread, "table.post") do
%{
userinfo: post |> userinfo(),
postdate: post |> postdate(),
postbody: post |> postbody()
}
end
%{id: id, title: title, page: page, page_count: page_count, posts: posts}
end
defp userinfo(post) do
user = Floki.find(post, "dl.userinfo")
name = user |> Floki.find("dt") |> Floki.text()
regdate = user |> Floki.find("dd.registered") |> Floki.text()
title = user |> Floki.find("dd.title") |> List.first() |> Floki.children() |> Floki.raw_html()
%{
name: name,
regdate: regdate,
title: title
}
end
defp postdate(post) do
date = Floki.find(post, "td.postdate") |> Floki.find("td.postdate") |> Floki.text()
[month_text, day, year, hours, minutes] =
date
|> String.split(~r{[\s,:]}, trim: true)
|> Enum.drop(1)
month =
1 +
Enum.find_index(
["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"],
fn m -> m == month_text end
)
NaiveDateTime.new!(
year |> String.to_integer(),
month,
day |> String.to_integer(),
hours |> String.to_integer(),
minutes |> String.to_integer(),
0
)
end
defp postbody(post) do
body =
Floki.find(post, "td.postbody")
|> List.first()
|> Floki.filter_out(:comment)
Floki.traverse_and_update(body, fn
{"img", attrs, []} -> transform(:img, attrs)
{"a", attrs, children} -> transform(:a, attrs, children)
other -> other
end)
|> Floki.children()
|> Floki.raw_html()
end
defp transform(elem, attr, children \\ [])
defp transform(:img, attrs, _children) do
{"class", class} = List.keyfind(attrs, "class", 0, {"class", ""})
if class == "sa-smilie" do
{"img", attrs, []}
else
t_attrs = List.keyreplace(attrs, "class", 0, {"class", "img-responsive"})
{"img", [{"loading", "lazy"} | t_attrs], []}
end
end
defp transform(:a, attrs, children) do
{"href", href} = List.keyfind(attrs, "href", 0, {"href", ""})
cond do
# skip internal links
String.starts_with?(href, "/") ->
{"a", [{"href", href}], children}
# mp4
String.ends_with?(href, ".mp4") ->
transform_link(:mp4, href)
# gifv
String.ends_with?(href, ".gifv") ->
transform_link(:gifv, href)
# youtube
String.starts_with?(href, "https://www.youtube.com/watch") ->
transform_link(:ytlong, href)
String.starts_with?(href, "https://youtu.be/") ->
transform_link(:ytshort, href)
true ->
Logger.debug("no transform for #{href}")
{"a", [{"href", href}], children}
end
end
defp transform_link(:mp4, href),
do:
{"div", [{"class", "responsive-embed"}],
[
{"video", [{"class", "img-responsive"}, {"controls", ""}],
[{"source", [{"src", href}, {"type", "video/mp4"}], []}]}
]}
defp transform_link(:gifv, href),
do:
{"div", [{"class", "responsive-embed"}],
[
{"video", [{"class", "img-responsive"}, {"controls", ""}],
[
{"source", [{"src", String.replace(href, ".gifv", ".webm")}, {"type", "video/webm"}],
[]},
{"source", [{"src", String.replace(href, ".gifv", ".mp4")}, {"type", "video/mp4"}],
[]}
]}
]}
defp transform_link(:ytlong, href) do
String.replace(href, "/watch?v=", "/embed/")
|> youtube_iframe()
end
defp transform_link(:ytshort, href) do
String.replace(href, "youtu.be/", "www.youtube.com/embed/")
|> youtube_iframe()
end
defp youtube_iframe(src),
do:
{"div", [{"class", "responsive-embed"}],
[
{"iframe",
[
{"class", "youtube-player"},
{"loading", "lazy"},
{"allow", "fullscreen"},
{"src", src}
], []}
]}
end

76
lib/something_erlang/grover.ex
View File

@ -1,76 +0,0 @@
defmodule SomethingErlang.Grover do
use GenServer
alias SomethingErlang.AwfulApi
require Logger
def mount(user) do
grover =
DynamicSupervisor.start_child(
SomethingErlang.Supervisor.Grovers,
{__MODULE__, [self(), user]}
)
case grover do
{:ok, pid} -> pid
{:error, {:already_started, pid}} -> pid
{:error, error} -> {:error, error}
end
end
def get_thread!(thread_id, page_number) do
GenServer.call(via(self()), {:show_thread, thread_id, page_number})
end
def get_bookmarks!(page_number) do
GenServer.call(via(self()), {:show_bookmarks, page_number})
end
def start_link([lv_pid, user]) do
GenServer.start_link(
__MODULE__,
[lv_pid, user],
name: via(lv_pid)
)
end
@impl true
def init([pid, user]) do
%{bbuserid: userid, bbpassword: userhash} = user
initial_state = %{
lv_pid: pid,
user: %{id: userid, hash: userhash}
}
Logger.debug("init #{userid} #{inspect(pid)}")
Process.monitor(pid)
{:ok, initial_state}
end
@impl true
def handle_call({:show_thread, thread_id, page_number}, _from, state) do
thread = AwfulApi.parsed_thread(thread_id, page_number, state.user)
{:reply, thread, state}
end
@impl true
def handle_call({:show_bookmarks, _page_number}, _from, state) do
bookmarks = AwfulApi.bookmarks(state.user)
{:reply, bookmarks, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, _object, reason}, state) do
Logger.debug("received :DOWN from: #{inspect(state.lv_pid)} reason: #{inspect(reason)}")
case reason do
{:shutdown, _} -> {:stop, :normal, state}
:killed -> {:stop, :normal, state}
_ -> {:noreply, state}
end
end
defp via(lv_pid),
do: {:via, Registry, {SomethingErlang.Registry.Grovers, lv_pid}}
end

28
lib/something_erlang/release.ex
View File

@ -1,28 +0,0 @@
defmodule SomethingErlang.Release do
@moduledoc """
Used for executing DB release tasks when run in production without Mix
installed.
"""
@app :something_erlang
def migrate do
load_app()
for repo <- repos() do
{:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true))
end
end
def rollback(repo, version) do
load_app()
{:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
end
defp repos do
Application.fetch_env!(@app, :ecto_repos)
end
defp load_app do
Application.load(@app)
end
end