diff --git a/packages/volto/news/5580.feature b/packages/volto/news/5580.feature new file mode 100644 index 0000000000..6e135f937f --- /dev/null +++ b/packages/volto/news/5580.feature @@ -0,0 +1 @@ +Expose robots.txt setting in Volto control panel, and render robots.txt based on REST API call. @robgietema \ No newline at end of file diff --git a/packages/volto/public/robots.txt b/packages/volto/public/robots.txt deleted file mode 100644 index 54fb98911d..0000000000 --- a/packages/volto/public/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * - diff --git a/packages/volto/src/config/ControlPanels.js b/packages/volto/src/config/ControlPanels.js index 1aaeabf488..3c408d99ef 100644 --- a/packages/volto/src/config/ControlPanels.js +++ b/packages/volto/src/config/ControlPanels.js @@ -76,7 +76,6 @@ export const unwantedControlPanelsFields = { 'site_favicon_mimetype', 'exposeDCMetaTags', 'enable_sitemap', - 'robots_txt', 'webstats_js', ], editing: ['available_editors', 'default_editor', 'ext_editor'], diff --git a/packages/volto/src/express-middleware/robotstxt.js b/packages/volto/src/express-middleware/robotstxt.js index b0fb708507..75d993afd1 100644 --- a/packages/volto/src/express-middleware/robotstxt.js +++ b/packages/volto/src/express-middleware/robotstxt.js @@ -4,13 +4,12 @@ import { generateRobots } from '@plone/volto/helpers/Robots/Robots'; /* robots.txt - priority order: -1) robots.txt in /public folder -2) VOLTO_ROBOTSTXT var in .env -3) default: plone robots.txt +1) VOLTO_ROBOTSTXT var in .env +2) robots.txt setting in the site control panel */ -const ploneRobots = function (req, res, next) { +const siteRobots = function (req, res, next) { generateRobots(req).then((robots) => { res.set('Content-Type', 'text/plain'); res.send(robots); @@ -27,7 +26,7 @@ export default function robotstxtMiddleware() { if (process.env.VOLTO_ROBOTSTXT) { middleware.all('**/robots.txt', envRobots); } else { - middleware.all('**/robots.txt', ploneRobots); + middleware.all('**/robots.txt', siteRobots); } middleware.id = 'robots.txt'; return middleware; diff --git a/packages/volto/src/helpers/Api/Api.js b/packages/volto/src/helpers/Api/Api.js index b92fd896c1..44e6a89123 100644 --- a/packages/volto/src/helpers/Api/Api.js +++ b/packages/volto/src/helpers/Api/Api.js @@ -17,7 +17,7 @@ const methods = ['get', 'post', 'put', 'patch', 'del']; * @param {string} path Path (or URL) to be formatted. * @returns {string} Formatted path. */ -function formatUrl(path) { +export function formatUrl(path) { const { settings } = config; const APISUFIX = settings.legacyTraverse ? '' : '/++api++'; diff --git a/packages/volto/src/helpers/Robots/Robots.js b/packages/volto/src/helpers/Robots/Robots.js index b2299ab2ba..c27b9c4f29 100644 --- a/packages/volto/src/helpers/Robots/Robots.js +++ b/packages/volto/src/helpers/Robots/Robots.js @@ -1,10 +1,12 @@ /** - * Sitemap helper. - * @module helpers/Sitemap + * Robots helper. + * @module helpers/Robots */ import superagent from 'superagent'; + import config from '@plone/volto/registry'; +import { formatUrl } from '@plone/volto/helpers/Api/Api'; import { addHeadersFactory } from '@plone/volto/helpers/Proxy/Proxy'; /** @@ -15,43 +17,22 @@ import { addHeadersFactory } from '@plone/volto/helpers/Proxy/Proxy'; */ export const generateRobots = (req) => new Promise((resolve) => { - const internalUrl = - config.settings.internalApiPath ?? - config.settings.devProxyToApiPath ?? - config.settings.apiPath; - const request = superagent.get(`${internalUrl}/robots.txt`); - request.set('Accept', 'text/plain'); + const request = superagent.get(formatUrl('@site')); + request.set('Accept', 'application/json'); const authToken = req.universalCookies.get('auth_token'); if (authToken) { request.set('Authorization', `Bearer ${authToken}`); } request.use(addHeadersFactory(req)); - request.end((error, { text }) => { + request.end((error, { text, body }) => { if (error) { resolve(text || error); } else { - // It appears that express does not take the x-forwarded headers into - // consideration, so we do it ourselves. - const { - 'x-forwarded-proto': forwardedProto, - 'x-forwarded-host': forwardedHost, - 'x-forwarded-port': forwardedPort, - } = req.headers; - const proto = forwardedProto ?? req.protocol; - const host = forwardedHost ?? req.get('Host'); - const portNum = forwardedPort ?? req.get('Port'); - const port = - (proto === 'https' && '' + portNum === '443') || - (proto === 'http' && '' + portNum === '80') - ? '' - : `:${portNum}`; - // Plone has probably returned the sitemap link with the internal url. - // If so, let's replace it with the current one. - const url = `${proto}://${host}${port}`; - text = text.replace(internalUrl, url); - // Replace the sitemap with the sitemap index. - text = text.replace('sitemap.xml.gz', 'sitemap-index.xml'); - resolve(text); + resolve( + body['plone.robots_txt'] + .replace('{portal_url}', config.settings.publicURL) + .replace('sitemap.xml.gz', 'sitemap-index.xml'), + ); } }); });