From d8177f38bd808ed253146358c863cdf497fe3c06 Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Sun, 9 Aug 2026 08:29:02 +0300 Subject: [PATCH] Add public search discovery metadata --- compose.yaml | 1 + config/config.exs | 1 + config/runtime.exs | 16 ++++ config/test.exs | 1 + docs/verification.md | 30 +++++++ lib/who_need_help_web.ex | 2 +- .../components/layouts/root.html.heex | 7 ++ .../controllers/page_controller.ex | 48 ++++++---- .../controllers/search_document_controller.ex | 89 +++++++++++++++++++ lib/who_need_help_web/endpoint.ex | 2 +- lib/who_need_help_web/router.ex | 2 + lib/who_need_help_web/search_metadata.ex | 29 ++++++ priv/gettext/default.pot | 18 ++-- priv/static/robots.txt | 5 -- .../controllers/page_controller_test.exs | 15 ++++ .../search_document_controller_test.exs | 40 +++++++++ 16 files changed, 275 insertions(+), 31 deletions(-) create mode 100644 lib/who_need_help_web/controllers/search_document_controller.ex create mode 100644 lib/who_need_help_web/search_metadata.ex delete mode 100644 priv/static/robots.txt create mode 100644 test/who_need_help_web/controllers/search_document_controller_test.exs diff --git a/compose.yaml b/compose.yaml index 089de88..6bf9ace 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,5 @@ x-app-environment: &app-environment + DEPLOYMENT_ENV: ${DEPLOYMENT_ENV:?Set DEPLOYMENT_ENV in .env} APP_ROLE: web DATABASE_URL: ${DATABASE_URL:?Set DATABASE_URL in .env} DATABASE_SOCKET_DIR: ${DATABASE_SOCKET_DIR:-} diff --git a/config/config.exs b/config/config.exs index 6e90caa..4c3c009 100644 --- a/config/config.exs +++ b/config/config.exs @@ -27,6 +27,7 @@ config :who_need_help, :scopes, ] config :who_need_help, + deployment_env: :development, ecto_repos: [WhoNeedHelp.Repo], generators: [timestamp_type: :utc_datetime, binary_id: true], app_role: :web, diff --git a/config/runtime.exs b/config/runtime.exs index 3394d9b..239a4e0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,5 +1,20 @@ import Config +deployment_env = + case System.get_env("DEPLOYMENT_ENV", "development") do + "development" -> + :development + + "test" -> + :test + + "production" -> + :production + + other -> + raise "DEPLOYMENT_ENV must be development, test, or production; got #{inspect(other)}" + end + app_role = case System.get_env("APP_ROLE", "web") do "web" -> :web @@ -46,6 +61,7 @@ rate_limit_policies = end config :who_need_help, + deployment_env: deployment_env, app_role: app_role, codex_session_id: System.get_env("CODEX_SESSION_ID", "not-configured"), map_tile_url: diff --git a/config/test.exs b/config/test.exs index 406a121..f068a23 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,6 +1,7 @@ import Config config :who_need_help, :handover_secret, "isolated-test-handover-secret" +config :who_need_help, :deployment_env, :test config :who_need_help, :tracking_presence_cleanup_grace_ms, 100 # Unit tests opt in to individual policies inside the relevant test. This keeps # unrelated examples independent from shared counters and mirrors the explicit diff --git a/docs/verification.md b/docs/verification.md index 1f55ca6..d250066 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -2322,3 +2322,33 @@ promoted. `cf7bacdf61ffb171ebac85e32dfcf12bc6972d59`; its application, database, and Mailpit remained healthy, and public readiness remained `ready`. The remote repository and hackathon-test deployment were not mutated. + +# 2026-08-09 local public search-discovery verification + +- The public home, Privacy Policy, Safety rules, and Terms of Service pages now + expose locale-aware canonical and reciprocal `en`/`uk`/`ru`/`x-default` + alternate links. Every other route defaults to `noindex, nofollow`, so + authenticated, support, legal, moderation, request, activity, profile, and + staff surfaces are not presented as public search results. +- `/sitemap.xml` contains exactly twelve absolute entries: the four public + pages in each of the three supported locales. It contains no account, + request, activity, support, legal, moderation, notification, or staff URL. +- `/robots.txt` is environment-aware. Production permits the public pages, + lists the private route families as disallowed, and advertises the absolute + sitemap URL. Development and test configurations return `Disallow: /`. +- Nineteen focused controller tests passed against an isolated temporary + PostgreSQL database. The complete isolated quality/security gate then passed + 455 ExUnit tests, all fourteen browser map-clustering tests, and every + configured compiler, formatting, xref, Credo, Sobelow, Dialyzer, + dependency, image, Compose, Helm, migration, rollback, and observability + check. The Debian 13.6 runtime-image scan reported zero detected + vulnerabilities. +- The isolated quality unit + `codex-heavy-wnh-quality-seo-rerun-20260809-20260809-082004-916736.service` + exited with status `0` after 4 minutes 47 seconds and reported a 210.9 MiB + memory peak. Its run-scoped images, containers, networks, and volumes were + removed. Two older quality database volumes remain referenced by their own + stopped containers and were not changed by this verification. +- This verification changed only the local checkout. The public Git remote, + production deployment, and frozen hackathon test deployment were not + changed. diff --git a/lib/who_need_help_web.ex b/lib/who_need_help_web.ex index b06699b..e9c44b6 100644 --- a/lib/who_need_help_web.ex +++ b/lib/who_need_help_web.ex @@ -18,7 +18,7 @@ defmodule WhoNeedHelpWeb do """ def static_paths, - do: ~w(assets fonts images favicon.ico manifest.webmanifest offline.html robots.txt sw.js) + do: ~w(assets fonts images favicon.ico manifest.webmanifest offline.html sw.js) def router do quote do diff --git a/lib/who_need_help_web/components/layouts/root.html.heex b/lib/who_need_help_web/components/layouts/root.html.heex index 4b3ae0d..4041ae9 100644 --- a/lib/who_need_help_web/components/layouts/root.html.heex +++ b/lib/who_need_help_web/components/layouts/root.html.heex @@ -16,6 +16,7 @@ + + <%= if canonical_url = assigns[:canonical_url] do %> + + <% end %> + <%= for {language, url} <- assigns[:language_alternates] || [] do %> + + <% end %> <.live_title default="Who Need Help" suffix={ diff --git a/lib/who_need_help_web/controllers/page_controller.ex b/lib/who_need_help_web/controllers/page_controller.ex index e0b3ba5..713fe0a 100644 --- a/lib/who_need_help_web/controllers/page_controller.ex +++ b/lib/who_need_help_web/controllers/page_controller.ex @@ -1,42 +1,60 @@ defmodule WhoNeedHelpWeb.PageController do use WhoNeedHelpWeb, :controller + alias WhoNeedHelpWeb.SearchMetadata + def home(conn, _params) do - render(conn, :home, - page_description: + assigns = + SearchMetadata.public_page_assigns("/") + |> Keyword.put( + :page_description, gettext( "Ask nearby volunteers for free medicine pickup or safe practical help, coordinate privately, and verify the handover." ) - ) + ) + + render(conn, :home, assigns) end def privacy(conn, _params) do - render(conn, :privacy, - page_title: gettext("Privacy Policy"), - page_description: + assigns = + SearchMetadata.public_page_assigns("/privacy") + |> Keyword.put(:page_title, gettext("Privacy Policy")) + |> Keyword.put( + :page_description, gettext( "How Who Need Help processes account, Google sign-in, location, communication, and safety data." ) - ) + ) + + render(conn, :privacy, assigns) end def safety(conn, _params) do - render(conn, :safety, - page_title: gettext("Safety rules"), - page_description: + assigns = + SearchMetadata.public_page_assigns("/safety") + |> Keyword.put(:page_title, gettext("Safety rules")) + |> Keyword.put( + :page_description, gettext( "Safety guidance for voluntary help, medicine pickup, roadside requests, location sharing, and in-person activities." ) - ) + ) + + render(conn, :safety, assigns) end def terms(conn, _params) do - render(conn, :terms, - page_title: gettext("Terms of Service"), - page_description: + assigns = + SearchMetadata.public_page_assigns("/terms") + |> Keyword.put(:page_title, gettext("Terms of Service")) + |> Keyword.put( + :page_description, gettext( "The conditions for using Who Need Help to coordinate voluntary help and social activities." ) - ) + ) + + render(conn, :terms, assigns) end end diff --git a/lib/who_need_help_web/controllers/search_document_controller.ex b/lib/who_need_help_web/controllers/search_document_controller.ex new file mode 100644 index 0000000..6d7a4fe --- /dev/null +++ b/lib/who_need_help_web/controllers/search_document_controller.ex @@ -0,0 +1,89 @@ +defmodule WhoNeedHelpWeb.SearchDocumentController do + use WhoNeedHelpWeb, :controller + + alias WhoNeedHelpWeb.SearchMetadata + + @public_paths ["/", "/privacy", "/safety", "/terms"] + @locales ~w(en uk ru) + @private_prefixes ~w( + /account/ + /activities + /admin + /analytics + /auth + /categories/proposals + /legal + /mobile + /moderation + /notifications + /people + /profile + /reports + /requests + /support + /users + ) + + def robots(conn, _params) do + body = + case Application.fetch_env!(:who_need_help, :deployment_env) do + :production -> production_robots() + _non_production -> "User-agent: *\nDisallow: /\n" + end + + conn + |> put_resp_content_type("text/plain", "utf-8") + |> put_resp_header("cache-control", "public, max-age=3600") + |> send_resp(:ok, body) + end + + def sitemap(conn, _params) do + entries = + for path <- @public_paths, + locale <- @locales do + sitemap_entry(path, locale) + end + + body = + "\n" <> + " + "xmlns:xhtml=\"http://www.w3.org/1999/xhtml\">\n" <> + Enum.join(entries, "") <> + "\n" + + conn + |> put_resp_content_type("application/xml", "utf-8") + |> put_resp_header("cache-control", "public, max-age=3600") + |> send_resp(:ok, body) + end + + defp production_robots do + disallowed = Enum.map_join(@private_prefixes, "", &"Disallow: #{&1}\n") + + "User-agent: *\nAllow: /\n" <> + disallowed <> + "Sitemap: #{SearchMetadata.absolute_url("/sitemap.xml")}\n" + end + + defp sitemap_entry(path, locale) do + alternates = + Enum.map_join(@locales, "", fn alternate_locale -> + ~s( \n) + end) <> + ~s( \n) + + " \n" <> + " #{xml_escape(SearchMetadata.localized_url(path, locale))}\n" <> + alternates <> + " \n" + end + + defp xml_escape(value) do + value + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + |> String.replace("\"", """) + |> String.replace("'", "'") + end +end diff --git a/lib/who_need_help_web/endpoint.ex b/lib/who_need_help_web/endpoint.ex index d7bea78..9649878 100644 --- a/lib/who_need_help_web/endpoint.ex +++ b/lib/who_need_help_web/endpoint.ex @@ -26,7 +26,7 @@ defmodule WhoNeedHelpWeb.Endpoint do from: :who_need_help, gzip: not code_reloading?, only: WhoNeedHelpWeb.static_paths(), - only_matching: ~w(favicon manifest offline robots sw), + only_matching: ~w(favicon manifest offline sw), raise_on_missing_only: code_reloading? # Code reloading can be explicitly enabled under the diff --git a/lib/who_need_help_web/router.ex b/lib/who_need_help_web/router.ex index 5175859..ca9bc98 100644 --- a/lib/who_need_help_web/router.ex +++ b/lib/who_need_help_web/router.ex @@ -79,6 +79,8 @@ defmodule WhoNeedHelpWeb.Router do scope "/", WhoNeedHelpWeb do pipe_through :browser + get "/robots.txt", SearchDocumentController, :robots + get "/sitemap.xml", SearchDocumentController, :sitemap get "/", PageController, :home get "/feedback", FeedbackController, :show get "/privacy", PageController, :privacy diff --git a/lib/who_need_help_web/search_metadata.ex b/lib/who_need_help_web/search_metadata.ex new file mode 100644 index 0000000..dbf0747 --- /dev/null +++ b/lib/who_need_help_web/search_metadata.ex @@ -0,0 +1,29 @@ +defmodule WhoNeedHelpWeb.SearchMetadata do + @moduledoc false + + @locales ~w(en uk ru) + + def public_page_assigns(path) when is_binary(path) do + locale = Gettext.get_locale(WhoNeedHelpWeb.Gettext) + + [ + page_robots: "index, follow", + canonical_url: localized_url(path, locale), + language_alternates: + Enum.map(@locales, &{&1, localized_url(path, &1)}) ++ + [{"x-default", absolute_url(path)}] + ] + end + + def localized_url(path, "en"), do: absolute_url(path) + + def localized_url(path, locale) when locale in @locales do + absolute_url(path) <> "?" <> URI.encode_query(%{"locale" => locale}) + end + + def localized_url(path, _unknown_locale), do: absolute_url(path) + + def absolute_url(path) do + String.trim_trailing(WhoNeedHelpWeb.Endpoint.url(), "/") <> path + end +end diff --git a/priv/gettext/default.pot b/priv/gettext/default.pot index 6969061..4dce6c3 100644 --- a/priv/gettext/default.pot +++ b/priv/gettext/default.pot @@ -1610,7 +1610,7 @@ msgid "Safety policy, category approval, account moderation, public launch, and msgstr "" #: lib/who_need_help_web/components/layouts.ex:191 -#: lib/who_need_help_web/controllers/page_controller.ex:25 +#: lib/who_need_help_web/controllers/page_controller.ex:36 #: lib/who_need_help_web/controllers/page_html/safety.html.heex:6 #, elixir-autogen, elixir-format msgid "Safety rules" @@ -1685,7 +1685,7 @@ msgstr "" msgid "Signal updated." msgstr "" -#: lib/who_need_help_web/components/layouts/root.html.heex:48 +#: lib/who_need_help_web/components/layouts/root.html.heex:55 #, elixir-autogen, elixir-format msgid "Skip to main content" msgstr "" @@ -3395,7 +3395,7 @@ msgid "Privacy" msgstr "" #: lib/who_need_help_web/controllers/google_auth_html/complete.html.heex:107 -#: lib/who_need_help_web/controllers/page_controller.ex:15 +#: lib/who_need_help_web/controllers/page_controller.ex:22 #: lib/who_need_help_web/controllers/page_html/privacy.html.heex:4 #: lib/who_need_help_web/controllers/page_html/terms.html.heex:75 #: lib/who_need_help_web/controllers/user_registration_html/new.html.heex:79 @@ -5442,7 +5442,7 @@ msgstr "" msgid "Restore request" msgstr "" -#: lib/who_need_help_web/controllers/page_controller.ex:27 +#: lib/who_need_help_web/controllers/page_controller.ex:39 #, elixir-autogen, elixir-format msgid "Safety guidance for voluntary help, medicine pickup, roadside requests, location sharing, and in-person activities." msgstr "" @@ -5926,7 +5926,7 @@ msgstr "" msgid "Account settings control profile, location visibility, direct messages, and connected Google sign-in. Live tracking is optional and can be stopped. You may submit a" msgstr "" -#: lib/who_need_help_web/controllers/page_controller.ex:7 +#: lib/who_need_help_web/controllers/page_controller.ex:11 #, elixir-autogen, elixir-format msgid "Ask nearby volunteers for free medicine pickup or safe practical help, coordinate privately, and verify the handover." msgstr "" @@ -6001,7 +6001,7 @@ msgstr "" msgid "Hosting, database, backup, transactional-email, and map-tile providers process the minimum data needed to provide their component of the service. External social-profile and thank-you links are controlled by their own operators and policies." msgstr "" -#: lib/who_need_help_web/controllers/page_controller.ex:17 +#: lib/who_need_help_web/controllers/page_controller.ex:25 #, elixir-autogen, elixir-format msgid "How Who Need Help processes account, Google sign-in, location, communication, and safety data." msgstr "" @@ -6071,13 +6071,13 @@ msgstr "" msgid "Some safety, fraud, support, completed-help, backup, or legal records may need to remain, as explained in the Privacy Policy. Updated terms will be published here with a new effective date; material changes should be reviewed before continuing to use the service." msgstr "" -#: lib/who_need_help_web/controllers/page_controller.ex:35 +#: lib/who_need_help_web/controllers/page_controller.ex:50 #: lib/who_need_help_web/controllers/page_html/terms.html.heex:4 #, elixir-autogen, elixir-format msgid "Terms of Service" msgstr "" -#: lib/who_need_help_web/controllers/page_controller.ex:37 +#: lib/who_need_help_web/controllers/page_controller.ex:53 #, elixir-autogen, elixir-format msgid "The conditions for using Who Need Help to coordinate voluntary help and social activities." msgstr "" @@ -6432,7 +6432,7 @@ msgstr "" msgid "Waiting for Android notification permission and token…" msgstr "" -#: lib/who_need_help_web/components/layouts/root.html.heex:23 +#: lib/who_need_help_web/components/layouts/root.html.heex:24 #, elixir-autogen, elixir-format msgid "Who Need Help connects adults who need urgent local help with nearby volunteers." msgstr "" diff --git a/priv/static/robots.txt b/priv/static/robots.txt deleted file mode 100644 index 26e06b5..0000000 --- a/priv/static/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / diff --git a/test/who_need_help_web/controllers/page_controller_test.exs b/test/who_need_help_web/controllers/page_controller_test.exs index d18042c..1164de4 100644 --- a/test/who_need_help_web/controllers/page_controller_test.exs +++ b/test/who_need_help_web/controllers/page_controller_test.exs @@ -5,6 +5,7 @@ defmodule WhoNeedHelpWeb.PageControllerTest do conn = get(conn, ~p"/") html = html_response(conn, 200) document = LazyHTML.from_document(html) + endpoint_url = WhoNeedHelpWeb.Endpoint.url() [content_security_policy] = get_resp_header(conn, "content-security-policy") assert html =~ "Need help nearby? Ask the community." @@ -18,6 +19,13 @@ defmodule WhoNeedHelpWeb.PageControllerTest do assert html =~ "Before you create a request" assert html =~ ~r/]*>Who Need Help<\/title>/ refute html =~ "Who Need Help · Who Need Help" + assert html =~ ~s( get(~p"/users/log-in") |> html_response(200) + + assert html =~ ~s(/, body)) == 12 + assert body =~ "#{endpoint_url}/" + assert body =~ "#{endpoint_url}/privacy?locale=uk" + assert body =~ "#{endpoint_url}/safety?locale=ru" + assert body =~ ~s(hreflang="x-default" href="#{endpoint_url}/terms") + refute body =~ "/users/" + refute body =~ "/requests" + refute body =~ "/support" + end + + test "non-production robots blocks crawling", %{conn: conn} do + conn = get(conn, ~p"/robots.txt") + + assert response(conn, 200) == "User-agent: *\nDisallow: /\n" + assert get_resp_header(conn, "content-type") == ["text/plain; charset=utf-8"] + end + + test "production robots advertises sitemap and avoids private route families", %{conn: conn} do + previous = Application.fetch_env!(:who_need_help, :deployment_env) + Application.put_env(:who_need_help, :deployment_env, :production) + on_exit(fn -> Application.put_env(:who_need_help, :deployment_env, previous) end) + + body = conn |> get(~p"/robots.txt") |> response(200) + + assert body =~ "Allow: /\n" + assert body =~ "Disallow: /admin\n" + assert body =~ "Disallow: /requests\n" + assert body =~ "Disallow: /support\n" + assert body =~ "Sitemap: #{WhoNeedHelpWeb.Endpoint.url()}/sitemap.xml\n" + end +end