Add skill update script (opsdroid#370)

* Add script for updating skill section in example configuration

* Remove creds from script

* Split script into functions

* Add fail strategy

* Fix config regex

* Rewrite update_config script using template approach

* PEP8 fix

* Fix flake8 error

* Restructure files and add README/docs

* Add comment about dev deps.

docs/contributing.md

@@ -66,6 +66,12 @@ Running the tests
 docker run --rm -ti -v $(pwd):/usr/src/app opsdroid/opsdroid:myfeature tox
 ```
 
+## Maintainer scripts
+
+This project contains a directory called [`scripts`](https://github.com/opsdroid/opsdroid/tree/master/scripts) which are simple python scripts for use by maintainers when working on opsdroid. Each directory contains the script itself, a README and other supporting files. See the individual README files for more information.
+
+_These scripts may have dependancies so you should run `pip install -r requirements_dev.txt` from the root of the project._
+
 ## Documentation
 More documentation is always appreciated and it's something that you can contribute to from the GitHub web interface.  This might be a great start point if you are new to Open Source and GitHub!
 

requirements_dev.txt

@@ -0,0 +1,2 @@
+PyGithub==1.35
+Jinja2==2.10

scripts/update_example_config/README.md

@@ -0,0 +1,3 @@
+# Update example configuration file
+
+This script uses the GitHub api to pull the configuration options for all official skill modules and updates the example configuration file with them all commented out.

scripts/update_example_config/configuration.j2

@@ -0,0 +1,114 @@
+##                      _           _     _
+##   ___  _ __  ___  __| |_ __ ___ (_) __| |
+##  / _ \| '_ \/ __|/ _` | '__/ _ \| |/ _` |
+## | (_) | |_) \__ \ (_| | | | (_) | | (_| |
+##  \___/| .__/|___/\__,_|_|  \___/|_|\__,_|
+##       |_|
+##                   __ _
+##   ___ ___  _ __  / _(_) __ _
+##  / __/ _ \| '_ \| |_| |/ _` |
+## | (_| (_) | | | |  _| | (_| |
+##  \___\___/|_| |_|_| |_|\__, |
+##                        |___/
+##
+## A default config file to use with opsdroid
+
+## Set the logging level
+# logging:
+#   level: info
+#   path: opsdroid.log
+#   console: true
+
+## Set the location for opsdroid to install modules
+# module-path: "."
+
+## Show welcome message
+welcome-message: true
+
+## Configure the web server
+# web:
+#   host: '127.0.0.1'
+#   port: 8080
+#   ssl:
+#     cert: /path/to/cert.pem
+#     key: /path/to/key.pem
+
+## Parsers
+# parsers:
+#   - name: regex
+#     enabled: true
+#
+#   - name: crontab
+#     enabled: true
+#
+#   - name: dialogflow
+#     access-token: "youraccesstoken"
+#     min-score: 0.6
+
+## Connector modules
+connectors:
+  - name: shell
+  - name: websocket
+
+  # Uncomment the connector(s) that you wish opsdroid to work on
+#  ## Slack (https://github.com/opsdroid/connector-slack)
+#  - name: slack
+#    # Required
+#    api-token: "zyxw-abdcefghi-12345"
+#    # optional
+#    bot-name: "mybot"       # default "opsdroid"
+#    default-room: "#random" # default "#general"
+#    icon-imoji: ":smile:"   # default ":robot_face:"
+#
+#  ## Facebook (https://github.com/opsdroid/connector-facebook)
+#  - name: facebook
+#    # Required
+#    verify-token: aabbccddee
+#    page-access-token: aabbccddee112233445566
+#    # Optional
+#    bot-name: "mybot" # default "opsdroid"
+#
+#  ## Twitter (https://github.com/opsdroid/connector-twitter)
+#  - name: twitter
+#    # Required
+#    consumer_key: "zyxw-abdcefghi-12345"
+#    consumer_secret: "zyxw-abdcefghi-12345-zyxw-abdcefghi-12345"
+#    oauth_token: ""zyxw-abdcefghi-12345-zyxw-abdcefghi-12345"
+#    oauth_token_secret: "zyxw-abdcefghi-12345-zyxw-abdcefghi-12345"
+#    # Optional
+#    enable_dms: true    # Should the bot respond to Direct Messages
+#    enable_tweets: true # Should the bot respond to tweets
+#
+#  ## Telegram (https://github.com/opsdroid/connector-telegram)
+#  - name: telegram
+#    # Required
+#    token: "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ-ZYXWVUT"  # Telegram bot token
+#    # Optional
+#    update_interval: 0.5  # Interval between checking for messages
+#    default_user: user1   # Default user to send messages to (overrides default room in connector)
+#    whitelisted_users:    # List of users who can speak to the bot, if not set anyone can speak
+#      - user1
+#      - user2
+
+## Database modules (optional)
+# databases:
+#   - name: mongo
+#     host:       "my host"     # (Optional) default "localhost"
+#     port:       "12345"       # (Optional) default "27017"
+#     database:   "mydatabase"  # (Optional) default "opsdroid"
+
+## Skill modules
+skills:
+{% for skill in uncommented_skills %}
+  ## {{ skill.name }} ({{ skill.url }})
+  {% for line in skill.config.split('\n') -%}
+  {{ line }}
+  {% endfor -%}
+{% endfor %}
+  # Configurations for other skills uncomment desired skill to activate it.
+#{% for skill in commented_skills %}
+#  ## {{ skill.name }} ({{ skill.url }})
+{% for line in skill.config.split('\n') -%}
+#  {{ line }}
+{% endfor -%}
+#{% endfor %}

scripts/update_example_config/update_example_config.py

@@ -0,0 +1,150 @@
+from github import Github
+from argparse import ArgumentParser
+import jinja2
+import base64
+import yaml
+import re
+import os
+
+
+def normalize(string):
+    lines = string.strip().split('\n')
+    if 'skills' in lines[0]:
+        return '\n'.join([
+            re.sub('^(#)?  ', '\g<1>', line)
+            for line in lines[1:]
+        ])
+    return string.strip()
+
+
+def render(tpl_path, context):
+    path, filename = os.path.split(tpl_path)
+    return jinja2.Environment(
+        loader=jinja2.FileSystemLoader(path or './')
+    ).get_template(filename).render(context)
+
+
+def get_repos():
+    return [
+        repo
+        for repo in g.get_user("opsdroid").get_repos()
+        if repo.name.startswith('skill-')
+    ]
+
+
+def get_readme(repo):
+    readme_base64 = repo.get_readme().content
+    return base64.b64decode(readme_base64).decode("utf-8")
+
+
+def get_skill(repo, readme):
+    config = re.search(
+        '#[#\s]+Configuration((.|\n)*?)```(yaml)?\n((.|\n)*?)\n```',
+        readme,
+        re.MULTILINE
+    )
+
+    skill_raw_name = repo.name[6:]
+    skill_name = skill_raw_name.replace('-', ' ').capitalize()
+    skill_url = repo.html_url
+
+    if config:
+        skill_config = normalize(config.group(4))
+    else:
+        skill_config = '- name: ' + skill_raw_name
+
+    return {
+        'raw_name': skill_raw_name,
+        'name': skill_name,
+        'url': skill_url,
+        'config': skill_config
+    }
+
+
+def check_skill(repo, skill, error_strict):
+    try:
+        yaml.load(skill['config'])
+    except yaml.scanner.ScannerError as e:
+        if error_strict:
+            raise(e)
+        print(
+            "[WARNING] processing {0} raised an exception\n"
+            "{2}\n{1}\n{2}".format(repo.name, e, '='*40)
+        )
+
+
+def get_skills(g, active_skills, error_strict=False):
+    repos = get_repos()
+
+    skills = {'commented_skills': [], 'uncommented_skills': []}
+
+    for repo in repos:
+        readme = get_readme(repo)
+        skill = get_skill(repo, readme)
+        check_skill(repo, skill, error_strict)
+
+        if skill['raw_name'] in active_skills:
+            skills['uncommented_skills'].append(skill)
+        else:
+            skills['commented_skills'].append(skill)
+
+    return skills
+
+
+def check_config(config, error_strict):
+    try:
+        yaml.load(config)
+    except yaml.scanner.ScannerError as e:
+        if error_strict:
+            raise(e)
+        print(
+            "[WARNING] processing resulting config raised an exception"
+            "\n{1}\n{0}\n{1}".format(e, '='*40)
+        )
+
+
+def update_config(g, active_skills, config_path, error_strict=False):
+    skills = get_skills(g, active_skills, error_strict)
+    text = render('scripts/configuration.j2', skills)
+    check_config(text, error_strict)
+
+    with open(config_path, 'w') as f:
+        f.write(text)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='Config creator ')
+    parser.add_argument('output', nargs='?', help='Path to config to update')
+    parser.add_argument('-t', '--token', nargs='?', help='GitHub Token')
+    parser.add_argument('-a', '--active-skills',
+                        nargs='?', help='List of skills to be activated')
+
+    parser.set_defaults(error_strict=False)
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--strict', dest='error_strict', action='store_true',
+        help='Sets fail strategy to strict mode. Fails on any error.'
+    )
+    group.add_argument(
+        '--warn', dest='error_strict', action='store_false',
+        help='Sets fail strategy to warn mode (default).'
+        ' Any errors are shown as warnings.'
+    )
+
+    args = parser.parse_args()
+
+    g = Github(args.token)
+
+    if args.active_skills:
+        active_skills = args.active_skills.split(',')
+    else:
+        active_skills = ['dance', 'hello', 'seen', 'loudnoises']
+
+    if not args.output:
+        base_path = '/'.join(os.path.realpath(__file__).split('/')[:-2])
+        config_path = base_path
+        config_path += '/opsdroid/configuration/example_configuration.yaml'
+    else:
+        config_path = args.output
+
+    update_config(g, active_skills, config_path, args.error_strict)