add robots.txt and copy it to public folder during build

cypress/integration/changelog_spec.js

@@ -9,7 +9,7 @@ describe('Changelog', () => {
     .should('be.visible')
   )
 
-  if (Cypress.env('NODE_ENV') === 'development') {
+  if (Cypress.isDevelopment()) {
     it('has a truncated table of contents', () => {
       cy.get('aside#article-toc')
       .should('be.visible')

cypress/integration/main_spec.js

@@ -1,10 +1,24 @@
+// loads Cypress types and any of our custom commands
+/// <reference path="../support/index.d.ts" />
+// @ts-check
+
 const GUIDES_PATH = '/guides/overview/why-cypress.html'
 
 describe('Main', () => {
   beforeEach(() => {
     cy.server()
   })
 
+  context('robots.txt', () => {
+    if (!Cypress.isDevelopment()) {
+      it('has robots.txt', () => {
+        cy.request('/robots.txt').its('body')
+        .should('include', 'Disallow: /ja/')
+        .and('include', 'Disallow: /zh-cn/')
+      })
+    }
+  })
+
   context('CSS', () => {
     beforeEach(() => {
       cy.visit('/')
@@ -15,7 +29,7 @@ describe('Main', () => {
 
     // only works in development environment where each CSS
     // file is separate
-    if (Cypress.env('NODE_ENV') === 'development') {
+    if (Cypress.isDevelopment()) {
       it('loads roboto', () => {
         cy.request('/fonts/vendor/roboto-fontface/css/roboto/roboto-fontface.css')
       })

cypress/support/index.d.ts

@@ -0,0 +1,34 @@
+/// <reference types="cypress" />
+
+/**
+ * Possible environment names
+*/
+enum EnvironmentName {
+  development,
+  staging,
+  production
+}
+
+declare namespace Cypress {
+  interface Cypress {
+    /**
+     * Returns true if Cypress tests run against development environment.
+     */
+    isDevelopment (): boolean
+
+    /**
+     * Returns true if Cypress tests run against staging environment.
+     */
+    isStaging (): boolean
+
+    /**
+     * Returns true if Cypress tests run against production environment.
+     */
+    isProduction (): boolean
+
+    /**
+     * Returns true if Cypress tests run against this specific environment.
+     */
+    is (environmentName: EnvironmentName): boolean
+  }
+}

cypress/support/index.js

@@ -1,7 +1,18 @@
+/// <reference types="cypress" />
 import './defaults'
 
 const resizeObserverLoopErrRe = /^ResizeObserver loop limit exceeded/
 
+// a couple of shortcuts to check which environment we are running in
+
+Cypress.isDevelopment = () => Cypress.env('NODE_ENV') === 'development'
+
+Cypress.isStaging = () => Cypress.env('NODE_ENV') === 'staging'
+
+Cypress.isProduction = () => Cypress.env('NODE_ENV') === 'production'
+
+Cypress.is = (envName) => Cypress.env('NODE_ENV') === envName
+
 Cypress.on('uncaught:exception', (err) => {
   // https://github.com/WICG/ResizeObserver/issues/38
   if (resizeObserverLoopErrRe.test(err.message)) {

gulpfile.js

@@ -92,6 +92,13 @@ gulp.task('copy:tmp:to:public', () => {
   .pipe(gulp.dest('public'))
 })
 
+// move robots.txt
+gulp.task('move:robots.txt:to:public', function () {
+  return gulp
+  .src('./robots.txt*')
+  .pipe(gulp.dest('public'))
+})
+
 gulp.task('clean:non:application:js', () => {
   return remove('public/js/!(application).js')
 })
@@ -122,5 +129,5 @@ gulp.task('copy:static:assets', gulp.parallel('move:menu:spy:js', 'move:scrollin
 
 gulp.task('pre:build', gulp.parallel('copy:static:assets'))
 
-gulp.task('post:build', gulp.series('clean:js', 'clean:css', 'clean:fonts:folders', 'revision', 'clean:public', 'copy:tmp:to:public', 'clean:tmp'))
+gulp.task('post:build', gulp.series('clean:js', 'clean:css', 'clean:fonts:folders', 'revision', 'clean:public', 'copy:tmp:to:public', 'move:robots.txt:to:public', 'clean:tmp'))