added pytests for north american coverage + more housekeeping

Pipfile

@@ -7,7 +7,6 @@ verify_ssl = true
 
 [packages]
 jobfunnel = {path = ".",editable = true}
-nltk = "==3.4.5"
 
 [requires]
 python_version = "3.6"

demo/settings.yaml

@@ -1,17 +1,15 @@
-# All paths are relative to this file.
+# all paths are relative to this file
 
-# Paths.
-# place the search right next to this file
+# paths
 output_path: './'
 
-# Providers from which to search (case insensitive)
+# providers from which to search (case insensitive)
 providers:
   - 'Indeed'
   - 'Monster'
-  - 'GlassDoor' # This used to take ~10x longer to run than the other providers
-
+  - 'GlassDoor'
 
-# Filters.
+# filters
 search_terms:
   region:
     province: 'ON'
@@ -21,32 +19,29 @@ search_terms:
 
   keywords:
     - 'Python'
-    - 'Java'
 
 black_list:
   - 'Infox Consulting'
   - 'Terminal'
 
-# Logging level options are: critical, error, warning, info, debug, notset
+# logging level options are: critical, error, warning, info, debug, notset
 log_level: 'info'
 
-# Saves duplicates removed by tfidf filter to duplicate_list.csv
+# saves duplicates removed by tfidf filter to duplicate_list.csv
 save_duplicates: False
 
-# Turn on or off delaying
-set_delay: True 
+# turn on or off delaying
+set_delay: True
 
-# Delaying algorithm configuration
+# delaying algorithm configuration
 delay_config:
-    # Functions used for delaying algorithm, options are: constant, linear, sigmoid
+    # functions used for delaying algorithm, options are: constant, linear, sigmoid
     function: 'linear'
-    # Maximum delay/upper bound for converging random delay
-    delay: 30
-    # Minimum delay/lower bound for random delay  
-    min_delay: 15 
-    # Random delay
-    random: True 
-    # Converging random delay, only used if 'random' is set to True
-    converge: True 
-
-
+    # maximum delay/upper bound for converging random delay
+    delay: 10
+    # minimum delay/lower bound for random delay
+    min_delay: 1
+    # random delay
+    random: True
+    # converging random delay, only used if 'random' is set to True
+    converge: True

jobfunnel/__init__.py

@@ -1 +1 @@
-__version__ = '2.0.1'
+__version__ = '2.0.2'

jobfunnel/__main__.py

@@ -1,14 +1,16 @@
 #!python
 """main script, scrapes data off several listings, pickles it,
 and applies search filters"""
+from typing import Union
+
 from .config.parser import parse_config
 
 from .jobfunnel import JobFunnel
 from .indeed import Indeed
 from .monster import Monster
 from .glassdoor import GlassDoor
 
-providers = {'indeed': Indeed, 'monster': Monster, 'glassdoor': GlassDoor}
+PROVIDERS = {'indeed': Indeed, 'monster': Monster, 'glassdoor': GlassDoor}
 
 
 def main():
@@ -29,7 +31,7 @@ def main():
         jf.load_pickle(config)
     else:
         for p in config['providers']:
-            provider = providers[p](config)
+            provider: Union[GlassDoor, Monster, Indeed] = PROVIDERS[p](config)
             provider_id = provider.__class__.__name__
             try:
                 provider.scrape()

jobfunnel/config/parser.py

@@ -11,11 +11,10 @@
               'debug': logging.DEBUG, 'notset': logging.NOTSET}
 
 
-def _parse_cli():
-    """Parse the command line arguments.
+def parse_cli():
+    """ Parse the command line arguments.
 
     """
-
     parser = argparse.ArgumentParser(
         'CLI options take precedence over settings in the yaml file'
         'empty arguments are replaced by settings in the default yaml file')
@@ -122,7 +121,7 @@ def _parse_cli():
 
 
 def parse_config():
-    """Parse the JobFunnel configuration settings.
+    """ Parse the JobFunnel configuration settings.
 
     """
     # find the jobfunnel root dir
@@ -134,7 +133,7 @@ def parse_config():
     default_yaml = yaml.safe_load(open(default_yaml_path, 'r'))
 
     # parse the command line arguments
-    cli = _parse_cli()
+    cli = parse_cli()
 
     # parse the settings file for the line arguments
     given_yaml = None
@@ -182,6 +181,13 @@ def parse_config():
     if cli.keywords is not None:
         config['search_terms']['keywords'] = cli.keywords
 
+    # search term state is inserted as province if province does not already exist
+    if 'state' in config['search_terms']['region']:
+        if (config['search_terms']['region']['state'] is not None) and \
+                (config['search_terms']['region']['province'] is None):
+            config['search_terms']['region']['province'] = \
+                config['search_terms']['region']['state']
+
     # parse the blacklist
     config['black_list'] = default_yaml['black_list']
     if given_yaml_path is not None:
@@ -230,7 +236,7 @@ def parse_config():
         if given_yaml_path is not None:
             config['delay_config'] = given_yaml['delay_config']
 
-        # Cli options for delaying configuration
+        # cli options for delaying configuration
         if cli.function is not None:
             config['delay_config']['function'] = cli.function
         if cli.delay is not None:
@@ -242,7 +248,7 @@ def parse_config():
             if cli.converge is not None:
                 config['delay_config']['converge'] = cli.converge
 
-        # Converts function name to lower case in config
+        # converts function name to lower case in config
         config['delay_config']['function'] = \
             config['delay_config']['function'].lower()
     else:

jobfunnel/config/settings.yaml

@@ -1,17 +1,17 @@
 # This is the default settings file. Do not edit.
 
-# All paths are relative to this file.
+# all paths are relative to this file
 
-# Paths.
+# paths
 output_path: 'search'
 
-# Providers from which to search (case insensitive)
+# providers from which to search (case insensitive)
 providers:
   - 'Indeed'
   - 'Monster'
-  - 'GlassDoor' # This used to take ~10x longer to run than the other providers
+  - 'GlassDoor'
 
-# Filters.
+# filters
 search_terms:
   region:
     province: 'ON'
@@ -22,28 +22,28 @@ search_terms:
   keywords:
     - 'Python'
 
-# Black-listed company names
+# black-listed company names
 black_list:
   - 'Infox Consulting'
 
-# Logging level options are: critical, error, warning, info, debug, notset
+# logging level options are: critical, error, warning, info, debug, notset
 log_level: 'info'
 
-# Saves duplicates removed by tfidf filter to duplicate_list.csv
+# saves duplicates removed by tfidf filter to duplicate_list.csv
 save_duplicates: False
 
-# Turn on or off delaying
-set_delay: True 
+# turn on or off delaying
+set_delay: True
 
-# Delaying algorithm configuration
+# delaying algorithm configuration
 delay_config:
-    # Functions used for delaying algorithm, options are: constant, linear, sigmoid
+    # functions used for delaying algorithm, options are: constant, linear, sigmoid
     function: 'linear'
-    # Maximum delay/upper bound for converging random delay 
-    delay: 15 
-    # Minimum delay/lower bound for random delay
-    min_delay: 1 
-    # Random delay
+    # maximum delay/upper bound for converging random delay
+    delay: 10
+    # minimum delay/lower bound for random delay
+    min_delay: 1
+    # random delay
     random: False
-    # Converging random delay, only used if 'random' is set to True
-    converge: False 
+    # converging random delay, only used if 'random' is set to True
+    converge: False