[SDPAP-5916] Updated robots.txt generation so it's part of ripple-nux…

…t-tide, updated list of excludedPaths. (#1230)

* feat(ripple-nuxt-tide): Allow ripple-nuxt-tide to handle robots.txt, add new routes to disallowed, update reference nuxt.config.js

* feat(ripple-nuxt-tide): Allow different config types, add oauth to excludedPaths, update examples

* style(ripple-nuxt-tide): Remove trailing comma

* feat(create-ripple-app): Add robots to default nuxt.config.js file

Co-authored-by: David Featherston <david.featherstone@dpc.vic.gov.au>

examples/basic-examples/nuxt.config.js

@@ -96,6 +96,7 @@ export default {
         password: process.env.SEARCH_AUTH_PASSWORD
       },
       loadOnDemand: 1
-    }
+    },
+    robots: true
   }
 }

examples/reference/nuxt.config.js

@@ -45,11 +45,6 @@ export default {
   modules: [
     // https://www.npmjs.com/package/@dpc-sdp/ripple-nuxt-tide
     '@dpc-sdp/ripple-nuxt-tide',
-    // we want to always disallow bots on the example site
-    ['@nuxtjs/robots', {
-      UserAgent: '*',
-      Disallow: '/'
-    }],
     '@nuxtjs/gtm',
     ['@dpc-sdp/ripple-data-vic-api', {
       logLevel: ['development', 'test'].includes(process.env.NODE_ENV) ? 'development' : 'production',
@@ -184,6 +179,11 @@ export default {
       },
       loadOnDemand: 1 // 0 for previous load mode. If you have a custom search page before Ripple v1.5.7, you need small change your code to turn on this. A example: https://github.com/dpc-sdp/ripple/pull/630/files#diff-c797d3457e8f4ca26b5707a65bc76189R37
     },
-    cachePurgePattern: []
+    cachePurgePattern: [],
+    // Opting into robot.txt generation from ripple-nuxt-tide (@nuxtjs/robots)
+    // boolean | robots: true (use defaults)
+    // object | robots: { UserAgent: 'Googlebot', Disallow: '/private' }
+    // array | robots:  [{ UserAgent: 'Googlebot', Disallow: '/private' }, { UserAgent: 'msnbot', Disallow: '/draft' }]
+    robots: true
   }
 }

packages/create-ripple-app/template/nuxt/nuxt.config.js

@@ -118,6 +118,7 @@ export default {
         password: process.env.SEARCH_AUTH_PASSWORD
       },
       loadOnDemand: 1
-    }
+    },
+    robots: true
   }
 }

packages/ripple-nuxt-tide/README.md

@@ -61,7 +61,10 @@ You Can move some setting value into your env variables.
     // For devOps to set custom cache purge regex patterns if needs
     cachePurgePattern: [
       'your-custom-api-url-regex-pattern'
-    ]
+    ],
+    // Lets ripple-nuxt-tide manage the robots.txt file
+    robots: true // Optional. Accepts boolean i.e. true to use defaults, an object or an array or objects.
+  }
 }
 ```
 

packages/ripple-nuxt-tide/lib/module.js

@@ -116,6 +116,42 @@ const nuxtTide = function (moduleOptions) {
     })
   }
 
+  // Add robots.txt
+  if (options?.robots) {
+    // Setup excluded paths
+    let excludedPaths = [
+      '/js',
+      '/img',
+      '/_nuxt/*',
+      '/oauth/*',
+      '/preview/*',
+      '/share-link'
+    ]
+
+    if (process.env.LAGOON_ENVIRONMENT_TYPE !== 'production') {
+      excludedPaths.push('/')
+    }
+
+    let robots = [
+      { UserAgent: 'SemrushBot', Disallow: '/' },
+      { UserAgent: '*', Disallow: excludedPaths }
+    ]
+
+    // Add supplied options to default robot options
+    if (Array.isArray(options.robots)) {
+      robots.push(...options.robots)
+    } else if (typeof options.robots === 'object') {
+      robots.push(options.robots)
+    }
+
+    // Add sitemap, this only works when the build target is server
+    if (this.options.target === 'server') {
+      robots.push({ Sitemap: (req) => `https://${req.headers.host}/sitemap.xml` })
+    }
+
+    this.addModule(['@nuxtjs/robots', robots])
+  }
+
   // https://toor.co/blog/nuxtjs-smooth-scrolling-with-hash-links/
   this.options.router.scrollBehavior = async (to, from, savedPosition) => {
     if (savedPosition) {