Add robots.txt setting.

docs/source/configuration/environmentvariables.md

@@ -68,11 +68,6 @@ $ VOLTO_ROBOTSTXT="User-agent: *
 Disallow: /" yarn start
 ```
 
-```{note}
-If you want to use the `VOLTO_ROBOTSTXT` environment variable, make sure to
-delete the file `public/robots.txt` from your project.
-```
-
 ### DEBUG
 
 It will enable the log several logging points scattered through the Volto code. It uses the `volto:` namespace.

packages/volto/news/5580.feature

@@ -0,0 +1 @@
+Expose robots.txt setting in Volto control panel, and render robots.txt based on REST API call. @robgietema

packages/volto/src/config/ControlPanels.js

@@ -79,7 +79,6 @@ export const filterControlPanelsSchema = (controlpanel) => {
       'site_favicon_mimetype',
       'exposeDCMetaTags',
       'enable_sitemap',
-      'robots_txt',
       'webstats_js',
     ],
     editing: ['available_editors', 'default_editor', 'ext_editor'],

packages/volto/src/express-middleware/robotstxt.js

@@ -4,13 +4,12 @@ import { generateRobots } from '@plone/volto/helpers';
 /*
 robots.txt - priority order:
 
-1) robots.txt in /public folder
-2) VOLTO_ROBOTSTXT var in .env
-3) default: plone robots.txt
+1) VOLTO_ROBOTSTXT var in .env
+2) robots.txt setting in the site control panel
 
 */
 
-const ploneRobots = function (req, res, next) {
+const siteRobots = function (req, res, next) {
   generateRobots(req).then((robots) => {
     res.set('Content-Type', 'text/plain');
     res.send(robots);
@@ -27,7 +26,7 @@ export default function robotstxtMiddleware() {
   if (process.env.VOLTO_ROBOTSTXT) {
     middleware.all('**/robots.txt', envRobots);
   } else {
-    middleware.all('**/robots.txt', ploneRobots);
+    middleware.all('**/robots.txt', siteRobots);
   }
   middleware.id = 'robots.txt';
   return middleware;

packages/volto/src/helpers/Api/Api.js

@@ -17,7 +17,7 @@ const methods = ['get', 'post', 'put', 'patch', 'del'];
  * @param {string} path Path (or URL) to be formatted.
  * @returns {string} Formatted path.
  */
-function formatUrl(path) {
+export function formatUrl(path) {
   const { settings } = config;
   const APISUFIX = settings.legacyTraverse ? '' : '/++api++';
 

packages/volto/src/helpers/Robots/Robots.js

@@ -1,10 +1,12 @@
 /**
- * Sitemap helper.
- * @module helpers/Sitemap
+ * Robots helper.
+ * @module helpers/Robots
  */
 
 import superagent from 'superagent';
+
 import config from '@plone/volto/registry';
+import { formatUrl } from '@plone/volto/helpers/Api/Api';
 import { addHeadersFactory } from '@plone/volto/helpers/Proxy/Proxy';
 
 /**
@@ -15,41 +17,23 @@ import { addHeadersFactory } from '@plone/volto/helpers/Proxy/Proxy';
  */
 export const generateRobots = (req) =>
   new Promise((resolve) => {
-    const internalUrl =
-      config.settings.internalApiPath ?? config.settings.apiPath;
-    const request = superagent.get(`${internalUrl}/robots.txt`);
-    request.set('Accept', 'text/plain');
+    const request = superagent.get(formatUrl('@site'));
+    request.set('Accept', 'application/json');
     const authToken = req.universalCookies.get('auth_token');
     if (authToken) {
       request.set('Authorization', `Bearer ${authToken}`);
     }
     request.use(addHeadersFactory(req));
-    request.end((error, { text }) => {
+    request.end((error, { text, body }) => {
       if (error) {
         resolve(text || error);
       } else {
-        // It appears that express does not take the x-forwarded headers into
-        // consideration, so we do it ourselves.
-        const {
-          'x-forwarded-proto': forwardedProto,
-          'x-forwarded-host': forwardedHost,
-          'x-forwarded-port': forwardedPort,
-        } = req.headers;
-        const proto = forwardedProto ?? req.protocol;
-        const host = forwardedHost ?? req.get('Host');
-        const portNum = forwardedPort ?? req.get('Port');
-        const port =
-          (proto === 'https' && '' + portNum === '443') ||
-          (proto === 'http' && '' + portNum === '80')
-            ? ''
-            : `:${portNum}`;
-        // Plone has probably returned the sitemap link with the internal url.
-        // If so, let's replace it with the current one.
-        const url = `${proto}://${host}${port}`;
-        text = text.replace(internalUrl, url);
-        // Replace the sitemap with the sitemap index.
-        text = text.replace('sitemap.xml.gz', 'sitemap-index.xml');
-        resolve(text);
+        resolve(
+          body['plone.robots_txt'].replace(
+            '{portal_url}',
+            config.settings.publicURL,
+          ),
+        );
       }
     });
   });