Add custom MTurk qualification support (#493)

* Added custom qualification types JSON file support.
* Added type-checking on advanced qualifications.
* Added example advanced qualification file and config.
* set no advanced_quals_path by default
* add note to docs

Co-authored-by: Dave Eargle <dave@daveeargle.com>

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - user_utils.PsiTurkAuthorization should not allow empty username or password! (#492)
 
+### Added
+- Add custom MTurk qualification support (#493)
+
 ## [3.1.0]
 ### Added
 - ability to launch the experiment server from a subdirectory instead of just

doc/settings.rst

@@ -248,6 +248,22 @@ workers with that qualification already set will neither see your ad nor be able
 to accept your HIT. This is the recommended way of excluding participants who
 have performed other HITs for you from participating in your new HIT.
 
+.. _advanced_quals:
+
+advanced_quals_path
+~~~~~~~~~~~~~~~~~~~
+
+A path to a custom JSON qualifications file, where you can define your own
+MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
+
+__ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
+
+:type: ``path``
+
+Example::
+
+    advanced_quals_path = ./advanced_quals.json
+
 
 .. _hit_configuration_ad_url:
 

psiturk/amt_services.py

@@ -325,6 +325,9 @@ def configure_hit(self, hit_config):
         for qual_id in hit_config['block_qualification_ids']:
             quals.append(dict(QualificationTypeId=qual_id,
                               Comparator='DoesNotExist'))
+
+        for advanced_qual in hit_config['advanced_qualifications']:
+            quals.append(dict(advanced_qual))
 
         # Create a HIT type for this HIT.
         hit_type = self.mtc.create_hit_type(

psiturk/amt_services_wrapper.py

@@ -15,6 +15,7 @@
 import datetime
 import random
 import string
+import json
 from builtins import object
 from builtins import range
 from builtins import str
@@ -828,6 +829,18 @@ def _generate_hit_config(self, ad_url, num_workers, reward, duration, require_qu
         block_quals = self.config.get('HIT Configuration', 'block_quals', fallback=None)
         if block_quals:
             block_qualification_ids.extend(block_quals.split(','))
+
+        advanced_quals_path = self.config.get('HIT Configuration', 'advanced_quals_path', fallback=None)
+        advanced_qualifications = []
+        if advanced_quals_path:
+            with open(advanced_quals_path) as f:
+                advanced_qualifications = json.load(f)
+                if not isinstance(advanced_qualifications, list):
+                    raise PsiturkException(message=f'JSON file "{advanced_quals_path}" must be a list of dicts')
+                else:
+                    for el in advanced_qualifications:
+                        if not isinstance(el, dict):
+                            raise PsiturkException(message=f'JSON file "{advanced_quals_path}" must be a list of dicts')
 
         hit_config = {
             "ad_location": ad_url,
@@ -845,6 +858,7 @@ def _generate_hit_config(self, ad_url, num_workers, reward, duration, require_qu
             "require_master_workers": self.config.getboolean('HIT Configuration',
                                                              'require_master_workers'),
             "require_qualification_ids": require_qualification_ids,
-            "block_qualification_ids": block_qualification_ids
+            "block_qualification_ids": block_qualification_ids,
+            "advanced_qualifications": advanced_qualifications
         }
         return hit_config

psiturk/example/advanced_quals.json.sample

@@ -0,0 +1,11 @@
+[
+    {
+        "QualificationTypeId": "<SOME QUALIFICATION TYPE ID>",
+        "Comparator": "GreaterThan",
+        "IntegerValues": [65]
+    },
+    {
+        "QualificationTypeId": "<SOME QUALIFICATION TYPE ID>",
+        "Comparator": "Exists"
+    }
+]

psiturk/example/config.txt.sample

@@ -50,6 +50,12 @@
 # accepting this HIT
 ;block_quals =
 
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+;advanced_quals_path = 
+
 ## Hit Configuration - Ad Url ##################################################
 # Config settings for constructing the task's "landing page"
 #
@@ -74,7 +80,7 @@
 # you may uncomment and use `ad_url`. You may want to use this if your
 # experiment is served from a subdirectory off of the domain name. Otherwise,
 # leave this as-is.
-;;ad_url = %(ad_url_protocol)s://%(ad_url_host)s:%(ad_url_port)s/%(ad_url_route)s
+;ad_url = %(ad_url_protocol)s://%(ad_url_host)s:%(ad_url_port)s/%(ad_url_route)s
 
 ############################## Database Parameters #############################
 [Database Parameters]
@@ -107,7 +113,7 @@
 ;errorlog = server.log
 # For backwards compatibility, `logfile` is synonymous with `errorlog`. If
 # both are set, `errorlog` will be preferred over `logfile`.
-;;logfile = server.log
+;logfile = server.log
 
 # Log level for the psiturk gunicorn server
 ;loglevel = 2

scripts/create_sample_config_from_defaults.py

@@ -13,7 +13,7 @@
             f'# Example config file. Uncomment lines (remove the `;`)\n'
             f'# in order to override defaults.\n'
         )
-    if len(line) > 1 and line[0] not in ['#', '[']:  # every line will have at least an \n char
+    if len(line) > 1 and line[0] not in ['#', '[', ';']:  # every line will have at least an \n char
         line = f';{line}'
     new_lines.append(line)
     line_num += 1