BB-749 Update robots.txt.j2 to accept Allow rule and multiple values

CHANGELOG.md

@@ -1,6 +1,10 @@
+- Role: nginx
+  - Modified robots.txt.j2 to accept the Allow rule.
+  - Modified robots.txt.j2 to accept either a single string or a list of strings for agent, disallow, and allow.
+
 - Role: ecommerce
   - Added CORS_ORIGIN_WHITELIST and CORS_URLS_REGEX to allow selective CORS whitelisting of origins/urls. 
-
+  
 - common_vars
   - Added new overridable variable `COMMON_LMS_BASE_URL`.
 

playbooks/roles/nginx/templates/edx/app/nginx/robots.txt.j2

@@ -1,5 +1,28 @@
 {% for item in NGINX_ROBOT_RULES %}
+{% if item.agent is string %}
 User-agent: {{ item.agent }}
+{% else %}
+{% for agent in item.agent %}
+User-agent: {{ agent }}
+{% endfor %}
+{% endif %}
+{% if item.allow is defined %}
+{% if item.allow is string %}
+Allow: {{ item.allow }}
+{% else %}
+{% for allow in item.allow %}
+Allow: {{ allow }}
+{% endfor %}
+{% endif %}
+{% endif %}
+{% if item.disallow is defined %}
+{% if item.disallow is string %}
 Disallow: {{ item.disallow }}
+{% else %}
+{% for disallow in item.disallow %}
+Disallow: {{ disallow }}
+{% endfor %}
+{% endif %}
+{% endif %}
 
 {% endfor %}