450 lines
14 KiB
Elixir
450 lines
14 KiB
Elixir
defmodule WhoNeedHelp.ContentRemoval do
|
|
@moduledoc "Separate intake and review queue for electronic content-removal notices."
|
|
|
|
import Ecto.Query
|
|
|
|
alias WhoNeedHelp.Accounts
|
|
alias WhoNeedHelp.Accounts.{Scope, User}
|
|
alias WhoNeedHelp.ContentRemoval.{Notice, Notifier}
|
|
alias WhoNeedHelp.Mail.ContentRemovalEmailWorker
|
|
alias WhoNeedHelp.Pagination
|
|
alias WhoNeedHelp.PublicAccess
|
|
alias WhoNeedHelp.Repo
|
|
alias WhoNeedHelp.Trust
|
|
alias WhoNeedHelp.Trust.RateLimiter
|
|
|
|
@access_salt "content-removal-access"
|
|
@confirmation_salt "content-removal-confirmation"
|
|
@urgent_categories [
|
|
:non_consensual_intimate_media,
|
|
:child_sexual_abuse_material,
|
|
:threat_to_life_or_safety
|
|
]
|
|
|
|
def change_notice(%Notice{} = notice, attrs \\ %{}) do
|
|
Notice.submission_changeset(notice, attrs)
|
|
end
|
|
|
|
def create_notice(scope, regime, attrs) when regime in [:general, :dsa, :take_it_down],
|
|
do: create_notice(scope, regime, attrs, nil)
|
|
|
|
def create_notice(scope, regime, attrs, client_scope)
|
|
when regime in [:general, :dsa, :take_it_down] do
|
|
user = scope_user(scope)
|
|
attrs = normalize_keys(attrs)
|
|
attrs = maybe_use_user_email(attrs, user)
|
|
category = enum_value(Notice, :category, attrs["category"])
|
|
status = if category in @urgent_categories, do: :urgent_review, else: :open
|
|
|
|
response_due_at =
|
|
if regime == :take_it_down, do: DateTime.add(DateTime.utc_now(:second), 48, :hour)
|
|
|
|
with {:ok, _limits} <-
|
|
RateLimiter.check_many(rate_scopes(user, attrs["contact_email"], client_scope)) do
|
|
Repo.transact(fn ->
|
|
with {:ok, notice} <-
|
|
%Notice{
|
|
reference: unique_reference(),
|
|
regime: regime,
|
|
status: status,
|
|
response_due_at: response_due_at,
|
|
requester_id: user && user.id,
|
|
contact_verified_at: user && DateTime.utc_now(:second)
|
|
}
|
|
|> Notice.submission_changeset(attrs)
|
|
|> Repo.insert(),
|
|
{:ok, _audit} <-
|
|
Trust.audit(
|
|
user && user.id,
|
|
"content_removal_notice.created",
|
|
"content_removal_notice",
|
|
notice.id,
|
|
%{
|
|
"category" => to_string(notice.category),
|
|
"regime" => to_string(notice.regime),
|
|
"status" => to_string(notice.status)
|
|
}
|
|
),
|
|
{:ok, _jobs} <- enqueue_created_emails(notice) do
|
|
{:ok, notice}
|
|
end
|
|
end)
|
|
end
|
|
end
|
|
|
|
def list_for_user(%Scope{user: %User{id: user_id}}) do
|
|
Notice
|
|
|> where([notice], notice.requester_id == ^user_id)
|
|
|> order_by([notice], desc: notice.inserted_at, desc: notice.id)
|
|
|> Repo.all()
|
|
end
|
|
|
|
def get_for_user(%Scope{user: %User{id: user_id}}, id) do
|
|
with {:ok, id} <- Ecto.UUID.cast(id),
|
|
%Notice{} = notice <- Repo.get_by(Notice, id: id, requester_id: user_id) do
|
|
{:ok, notice}
|
|
else
|
|
_ -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
def get_by_access_token(id, token) when is_binary(token) do
|
|
with {:ok, id} <- Ecto.UUID.cast(id),
|
|
{:ok, ^id} <-
|
|
PublicAccess.verify(@access_salt, token, max_age: case_access_max_age_seconds()),
|
|
%Notice{} = notice <- Repo.get(Notice, id) do
|
|
if notice.contact_verified_at do
|
|
{:ok, notice}
|
|
else
|
|
verify_legacy_confirmation(notice, token)
|
|
end
|
|
else
|
|
_ -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
def get_by_access_token(_id, _token), do: {:error, :not_found}
|
|
|
|
def verify_by_confirmation_token(id, token) when is_binary(token) do
|
|
with {:ok, id} <- Ecto.UUID.cast(id),
|
|
{:ok, ^id} <-
|
|
PublicAccess.verify(@confirmation_salt, token,
|
|
max_age: contact_verification_max_age_seconds()
|
|
),
|
|
%Notice{} = notice <- Repo.get(Notice, id) do
|
|
verify_contact(notice)
|
|
else
|
|
_ -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
def verify_by_confirmation_token(_id, _token), do: {:error, :not_found}
|
|
|
|
def access_token(%Notice{id: id}) do
|
|
PublicAccess.sign(@access_salt, id, max_age: case_access_max_age_seconds())
|
|
end
|
|
|
|
def confirmation_token(%Notice{id: id}) do
|
|
PublicAccess.sign(@confirmation_salt, id, max_age: contact_verification_max_age_seconds())
|
|
end
|
|
|
|
def status_url(%Notice{} = notice) do
|
|
token = access_token(notice)
|
|
|
|
PublicAccess.url("/legal/content-removal/#{notice.id}?token=#{URI.encode_www_form(token)}")
|
|
end
|
|
|
|
def confirmation_url(%Notice{} = notice) do
|
|
token = confirmation_token(notice)
|
|
|
|
PublicAccess.url(
|
|
"/legal/content-removal/#{notice.id}/verify?token=#{URI.encode_www_form(token)}"
|
|
)
|
|
end
|
|
|
|
def paginate_for_staff(%Scope{user: user}, options \\ []) do
|
|
if Accounts.authorized?(user, :legal_view) do
|
|
limit = Pagination.limit(options)
|
|
cursor = Pagination.cursor(options)
|
|
|
|
Notice
|
|
|> visible_to_staff()
|
|
|> maybe_regime(Keyword.get(options, :regime))
|
|
|> maybe_status(Keyword.get(options, :status))
|
|
|> maybe_assignee(Keyword.get(options, :assigned_to_id), user.id)
|
|
|> maybe_search(Keyword.get(options, :search))
|
|
|> before(cursor)
|
|
|> order_by([notice], desc: notice.inserted_at, desc: notice.id)
|
|
|> limit(^(limit + 1))
|
|
|> preload([:requester, :reviewed_by, :assigned_to])
|
|
|> Repo.all()
|
|
|> Pagination.page(limit, &{&1.inserted_at, &1.id})
|
|
else
|
|
%Pagination.Page{}
|
|
end
|
|
end
|
|
|
|
def get_for_staff(%Scope{user: user}, id) do
|
|
with true <- Accounts.authorized?(user, :legal_view),
|
|
{:ok, id} <- Ecto.UUID.cast(id),
|
|
%Notice{} = notice <-
|
|
Notice
|
|
|> visible_to_staff()
|
|
|> where([notice], notice.id == ^id)
|
|
|> preload([:requester, :reviewed_by, :assigned_to])
|
|
|> Repo.one() do
|
|
{:ok, notice}
|
|
else
|
|
false -> {:error, :forbidden}
|
|
_ -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
def moderate(%Scope{user: moderator}, id, attrs) do
|
|
with {:ok, id} <- Ecto.UUID.cast(id),
|
|
true <- Accounts.authorized?(moderator, :legal_manage) do
|
|
with {:ok, attrs} <- normalize_assignment(normalize_keys(attrs), :legal_manage) do
|
|
attrs =
|
|
Map.merge(attrs, %{
|
|
"reviewed_at" => DateTime.utc_now(:second),
|
|
"reviewed_by_id" => moderator.id
|
|
})
|
|
|
|
Repo.transact(fn ->
|
|
notice =
|
|
Notice
|
|
|> visible_to_staff()
|
|
|> where([notice], notice.id == ^id)
|
|
|> lock("FOR UPDATE")
|
|
|> Repo.one()
|
|
|
|
if notice do
|
|
changeset = Notice.moderation_changeset(notice, attrs)
|
|
|
|
requester_update? =
|
|
Ecto.Changeset.changed?(changeset, :status) or
|
|
Ecto.Changeset.changed?(changeset, :resolution_note)
|
|
|
|
with {:ok, notice} <- Repo.update(changeset),
|
|
{:ok, _audit} <-
|
|
Trust.audit(
|
|
moderator.id,
|
|
"content_removal_notice.moderated",
|
|
"content_removal_notice",
|
|
notice.id,
|
|
%{
|
|
"status" => to_string(notice.status),
|
|
"assigned_to_id" => notice.assigned_to_id
|
|
}
|
|
),
|
|
{:ok, _job} <- maybe_enqueue_update(notice, requester_update?) do
|
|
{:ok, {Repo.preload(notice, :assigned_to, force: true), requester_update?}}
|
|
end
|
|
else
|
|
{:error, :not_found}
|
|
end
|
|
end)
|
|
|> normalize_moderation_result()
|
|
end
|
|
else
|
|
false -> {:error, :forbidden}
|
|
_ -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
defp normalize_moderation_result({:ok, {notice, _requester_update?}}), do: {:ok, notice}
|
|
defp normalize_moderation_result(result), do: result
|
|
|
|
defp verify_contact(%Notice{contact_verified_at: nil} = notice) do
|
|
Repo.transact(fn ->
|
|
current =
|
|
Notice
|
|
|> where([record], record.id == ^notice.id)
|
|
|> lock("FOR UPDATE")
|
|
|> Repo.one()
|
|
|
|
cond do
|
|
is_nil(current) ->
|
|
{:error, :not_found}
|
|
|
|
current.contact_verified_at ->
|
|
{:ok, {current, :already_verified}}
|
|
|
|
true ->
|
|
with {:ok, verified} <-
|
|
current
|
|
|> Ecto.Changeset.change(contact_verified_at: DateTime.utc_now(:second))
|
|
|> Repo.update(),
|
|
{:ok, _jobs} <- enqueue_verified_emails(verified) do
|
|
{:ok, {verified, :newly_verified}}
|
|
end
|
|
end
|
|
end)
|
|
|> notify_verified_notice()
|
|
end
|
|
|
|
defp verify_contact(%Notice{} = notice), do: {:ok, notice}
|
|
|
|
defp notify_verified_notice({:ok, {notice, :newly_verified}}) do
|
|
{:ok, notice}
|
|
end
|
|
|
|
defp notify_verified_notice({:ok, {notice, :already_verified}}), do: {:ok, notice}
|
|
defp notify_verified_notice(result), do: result
|
|
|
|
defp verify_legacy_confirmation(%Notice{id: id} = notice, token) do
|
|
case PublicAccess.verify(@access_salt, token, max_age: contact_verification_max_age_seconds()) do
|
|
{:ok, ^id} -> verify_contact(notice)
|
|
_error -> {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
defp contact_verification_max_age_seconds do
|
|
Application.fetch_env!(:who_need_help, :public_contact_verification_max_age_seconds)
|
|
end
|
|
|
|
defp case_access_max_age_seconds do
|
|
Application.fetch_env!(:who_need_help, :public_case_access_max_age_seconds)
|
|
end
|
|
|
|
defp enqueue_email(notice, kind) do
|
|
%{notice_id: notice.id, kind: to_string(kind)}
|
|
|> ContentRemovalEmailWorker.new()
|
|
|> Oban.insert()
|
|
end
|
|
|
|
defp enqueue_created_emails(%Notice{contact_email: email, contact_verified_at: nil} = notice)
|
|
when is_binary(email) and email != "" do
|
|
with {:ok, confirmation} <- enqueue_email(notice, :confirmation) do
|
|
{:ok, [confirmation]}
|
|
end
|
|
end
|
|
|
|
defp enqueue_created_emails(%Notice{} = notice), do: enqueue_verified_emails(notice)
|
|
|
|
defp enqueue_verified_emails(%Notice{} = notice) do
|
|
with {:ok, received} <- maybe_enqueue_received(notice),
|
|
{:ok, operator} <- maybe_enqueue_operator(notice) do
|
|
{:ok, [received, operator]}
|
|
end
|
|
end
|
|
|
|
defp maybe_enqueue_received(%Notice{contact_email: email} = notice)
|
|
when is_binary(email) and email != "",
|
|
do: enqueue_email(notice, :received)
|
|
|
|
defp maybe_enqueue_received(%Notice{}), do: {:ok, :no_contact_email}
|
|
|
|
defp maybe_enqueue_operator(%Notice{} = notice) do
|
|
if Notifier.operator_alerts_enabled?(),
|
|
do: enqueue_email(notice, :operator),
|
|
else: {:ok, :disabled}
|
|
end
|
|
|
|
defp maybe_enqueue_update(%Notice{contact_verified_at: nil}, _requester_update?),
|
|
do: {:ok, :contact_not_verified}
|
|
|
|
defp maybe_enqueue_update(%Notice{} = notice, true), do: enqueue_email(notice, :update)
|
|
defp maybe_enqueue_update(%Notice{}, false), do: {:ok, :not_needed}
|
|
|
|
defp visible_to_staff(query) do
|
|
where(
|
|
query,
|
|
[notice],
|
|
is_nil(notice.contact_email) or not is_nil(notice.contact_verified_at)
|
|
)
|
|
end
|
|
|
|
defp before(query, nil), do: query
|
|
|
|
defp before(query, {inserted_at, id}) do
|
|
where(
|
|
query,
|
|
[notice],
|
|
notice.inserted_at < ^inserted_at or
|
|
(notice.inserted_at == ^inserted_at and notice.id < ^id)
|
|
)
|
|
end
|
|
|
|
defp maybe_regime(query, nil), do: query
|
|
defp maybe_regime(query, ""), do: query
|
|
defp maybe_regime(query, regime), do: where(query, [notice], notice.regime == ^regime)
|
|
|
|
defp maybe_status(query, nil), do: query
|
|
defp maybe_status(query, ""), do: query
|
|
defp maybe_status(query, status), do: where(query, [notice], notice.status == ^status)
|
|
|
|
defp maybe_assignee(query, nil, _current_user_id), do: query
|
|
defp maybe_assignee(query, "", _current_user_id), do: query
|
|
|
|
defp maybe_assignee(query, "unassigned", _current_user_id),
|
|
do: where(query, [n], is_nil(n.assigned_to_id))
|
|
|
|
defp maybe_assignee(query, "mine", current_user_id),
|
|
do: where(query, [n], n.assigned_to_id == ^current_user_id)
|
|
|
|
defp maybe_assignee(query, assignee_id, _current_user_id),
|
|
do: where(query, [n], n.assigned_to_id == ^assignee_id)
|
|
|
|
defp maybe_search(query, value) when value in [nil, ""], do: query
|
|
|
|
defp maybe_search(query, value) do
|
|
term = value |> String.trim() |> String.replace("%", "") |> String.replace("_", "")
|
|
|
|
if String.length(term) < 3 do
|
|
where(query, [notice], false)
|
|
else
|
|
pattern = "%#{String.downcase(term)}%"
|
|
|
|
where(
|
|
query,
|
|
[notice],
|
|
fragment(
|
|
"lower(coalesce(?, '') || ' ' || coalesce(?, '') || ' ' || coalesce(?, '')) LIKE ?",
|
|
notice.reference,
|
|
notice.contact_email,
|
|
notice.submitter_name,
|
|
^pattern
|
|
)
|
|
)
|
|
end
|
|
end
|
|
|
|
defp normalize_assignment(attrs, permission) do
|
|
case Map.fetch(attrs, "assigned_to_id") do
|
|
:error ->
|
|
{:ok, attrs}
|
|
|
|
{:ok, value} when value in [nil, ""] ->
|
|
{:ok, Map.put(attrs, "assigned_to_id", nil)}
|
|
|
|
{:ok, user_id} ->
|
|
if Accounts.authorized_user_id?(user_id, permission),
|
|
do: {:ok, attrs},
|
|
else: {:error, :invalid_assignee}
|
|
end
|
|
end
|
|
|
|
defp maybe_use_user_email(attrs, %User{email: email}) do
|
|
Map.put(attrs, "contact_email", email)
|
|
end
|
|
|
|
defp maybe_use_user_email(attrs, nil), do: attrs
|
|
|
|
defp rate_scopes(user, contact_email, client_scope) do
|
|
[{:content_removal_notice, rate_scope(user, contact_email)}]
|
|
|> maybe_add_client_rate_scope(client_scope)
|
|
end
|
|
|
|
defp maybe_add_client_rate_scope(scopes, client_scope)
|
|
when is_binary(client_scope) and client_scope != "",
|
|
do: [{:content_removal_notice_ip, client_scope} | scopes]
|
|
|
|
defp maybe_add_client_rate_scope(scopes, _client_scope), do: scopes
|
|
|
|
defp rate_scope(%User{id: id}, _email), do: "user:#{id}"
|
|
|
|
defp rate_scope(nil, email) when is_binary(email),
|
|
do: "email:#{String.downcase(String.trim(email))}"
|
|
|
|
defp rate_scope(nil, _email), do: "missing-email"
|
|
|
|
defp scope_user(%Scope{user: %User{} = user}), do: user
|
|
defp scope_user(_scope), do: nil
|
|
|
|
defp enum_value(module, field, value) do
|
|
Ecto.Enum.cast_value(module, field, value)
|
|
|> case do
|
|
{:ok, enum} -> enum
|
|
:error -> nil
|
|
end
|
|
end
|
|
|
|
defp unique_reference do
|
|
suffix = 8 |> :crypto.strong_rand_bytes() |> Base.encode32(case: :upper, padding: false)
|
|
"REM-#{suffix}"
|
|
end
|
|
|
|
defp normalize_keys(attrs), do: Map.new(attrs, fn {key, value} -> {to_string(key), value} end)
|
|
end
|