Disallow /live/longpoll in the generated robots.txt

assets/js/phoenix/socket.js

@@ -36,6 +36,11 @@ import Timer from "./timer"
  * @param {number} [opts.longPollFallbackMs] - The millisecond time to attempt the primary transport
  * before falling back to the LongPoll transport. Disabled by default.
  *
+ * Clients without WebSocket support fall back to LongPoll, and this includes search
+ * engine crawlers, whose renderers do not open WebSockets. They then repeatedly request
+ * the LongPoll endpoint (e.g. `/live/longpoll` for LiveView), which serves no indexable
+ * content. Disallow that path in `robots.txt` to avoid spending crawl budget on it.
+ *
  * @param {boolean} [opts.debug] - When true, enables debug logging. Default false.
  *
  * @param {Function} [opts.encode] - The function to encode outgoing messages.

installer/templates/phx_static/robots.txt

@@ -3,3 +3,9 @@
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-agent: *
 # Disallow: /
+
+# /live/longpoll is LiveView's fallback transport for clients that can't open a
+# WebSocket, which includes search engine crawlers. It carries no indexable
+# content, so disallow it to keep crawlers from spending crawl budget there.
+User-agent: *
+Disallow: /live/longpoll

installer/test/phx_new_test.exs

@@ -141,6 +141,7 @@ defmodule Mix.Tasks.Phx.NewTest do
 
       # assets
       assert_file("phx_blog/priv/static/images/logo.svg")
+      assert_file("phx_blog/priv/static/robots.txt", ~r"Disallow: /live/longpoll")
 
       assert_file("phx_blog/.gitignore", fn file ->
         assert file =~ "/priv/static/assets/"