Restructures project with proper imports. (#55)

nba_ss_db/__init__.py

@@ -0,0 +1,40 @@
+import yaml
+import os
+
+
+def load_config(file_name):
+    """
+    Loads the configurations located in the config file
+    at config/config.yaml
+    into the CONFIG dictionary.
+    """
+    CONFIG = {}
+    with open(file_name, 'r') as f:
+        config_yaml = yaml.load(f)
+        for config_key in config_yaml:
+            CONFIG[config_key] = config_yaml[config_key]
+    return CONFIG
+
+
+__location__ = os.path.realpath(os.path.join(
+    os.getcwd(), os.path.dirname(__file__)))
+CONFIG = load_config(os.path.join(__location__, 'config.yaml'))
+
+
+def _get_seasons():
+    """
+    Returns a list of strings representing seasons in the format:
+    '20XX-YY'
+    """
+    start_year = CONFIG['START_YEAR']
+    end_year = CONFIG['END_YEAR']
+
+    seasons = []
+    while start_year != end_year:
+        seasons.append('{}-{}'.format(start_year, (start_year + 1) % 100))
+        start_year += 1
+    return seasons
+
+
+SEASONS = _get_seasons()
+CONFIG['SEASONS'] = SEASONS

config/config.yaml → nba_ss_db/config.yaml

@@ -1,6 +1,6 @@
 # DB
 DB_NAME: 'nba_stats' # the name of the sqlite/db file
-DB_PATH: 'db/databases' # location of the db file
+DB_PATH: 'nba_ss_db/db/databases' # location of the db file
 IGNORE_DUPLICATES: True # Whether or not to include duplicate rows
 
 # Paths

nba_ss_db/db/__init__.py

@@ -0,0 +1 @@
+from . import initialize, request_logger, retrieve, store, utils

db/initialize.py → nba_ss_db/db/initialize.py

@@ -4,7 +4,7 @@
 to prevent double scraping.
 """
 
-import db.utils
+from nba_ss_db import db
 
 def init_db():
     """

db/request_logger.py → nba_ss_db/db/request_logger.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-import db.utils
+from nba_ss_db import db
 
 
 def log_request(api_request, table_name):

db/retrieve.py → nba_ss_db/db/retrieve.py

@@ -5,11 +5,11 @@
 
 The general query function will return a pandas dataframe.
 """
-import db.utils
+from nba_ss_db import db
 from collections import defaultdict
 import pandas as pd
 import os
-from config import CONFIG
+from nba_ss_db import CONFIG
 
 
 def fetch_player_ids():

db/store.py → nba_ss_db/db/store.py

@@ -2,9 +2,8 @@
 Handles the creation of tables and storage into tables.
 """
 from typing import List
-import db.retrieve
-import db.utils
-from config import CONFIG
+from nba_ss_db import db
+from nba_ss_db import CONFIG
 
 PROTECTED_COL_NAMES = {'TO'}
 

db/utils.py → nba_ss_db/db/utils.py

@@ -1,8 +1,8 @@
 import sqlite3
 import pandas as pd
-from config import CONFIG
+from nba_ss_db import CONFIG
 import datetime
-from scrape.utils import PROPER_DATE_FORMAT
+from nba_ss_db.scrape.utils import PROPER_DATE_FORMAT
 
 
 class DB_Query():

nba_ss_db/scrape/__init__.py

@@ -0,0 +1 @@
+from . import scraper, utils

scrape/scraper.py → nba_ss_db/scrape/scraper.py

@@ -5,10 +5,8 @@
 Ex. Scrape all ___ for each season for each position
 Ex. Scrape all ___ for each season for each player_id
 """
-import db.retrieve
-import db.store
-import db.request_logger
-from config import CONFIG
+from nba_ss_db import db
+from nba_ss_db import CONFIG
 
 from collections import OrderedDict
 
@@ -18,7 +16,7 @@
 import itertools
 import time
 import pprint
-from scrape.utils import *
+from nba_ss_db.scrape.utils import *
 
 from typing import Dict, List
 

run.py

@@ -1,7 +1,8 @@
 import argparse
-import scrape.scraper
-import db.utils
-import db.retrieve
+from nba_ss_db import db, scrape
+from nba_ss_db.scrape.scraper import run_scrape_jobs
+# from nba_ss_db.db.utils import drop_tables, clear_scrape_logs, get_db_connection, execute_sql_file_persist
+# from nba_ss_db.db.retrieve import df_to_csv
 
 parser = argparse.ArgumentParser(description='NBA Stats Scraper and Storage')
 parser.add_argument('--scrape', nargs='?', const='api_requests.yaml', dest='scrape_file_path',

setup.py

@@ -6,7 +6,7 @@
 setup(
     name='NBA Stats Scraper + DB Storage',
     version='0.0',
-    packages=['db', 'scrape'],
+    packages=['nba_ss_db'],
     description='A tool to automatically scrape and store stats.nba.com data into a database.',
     url='https://github.com/jsonchin/nba_stats_scraper_db_storage',
     license='MIT',

tests/test_db/test_db_retrieval.py

@@ -1,8 +1,7 @@
 import unittest
 from tests.test_setup import init_test_db
 
-import db.retrieve
-import db.utils
+from nba_ss_db import db
 import pandas as pd
 
 from collections import defaultdict

tests/test_db/test_db_storage.py

@@ -1,8 +1,7 @@
 import unittest
 from tests.test_setup import init_test_db
 
-import db.initialize
-import db.retrieve
+from nba_ss_db import db
 
 
 class TestDBStorage(unittest.TestCase):

tests/test_scrape/test_scraper.py

@@ -1,9 +1,7 @@
 import unittest
 
 from tests.test_setup import init_test_db
-
-import db.request_logger
-import db.utils
+from nba_ss_db import db
 
 
 class TestScraper(unittest.TestCase):

tests/test_setup.py

@@ -1,11 +1,11 @@
-from config import CONFIG
+from nba_ss_db import CONFIG
 CONFIG['DB_NAME'] = 'test_db'
 CONFIG['DB_PATH'] = 'tests/test_db/databases'
-from db.initialize import init_db
-import db.utils
+from nba_ss_db import db
+
 
 def init_test_db():
     db.utils.drop_tables()
-    init_db()
+    db.initialize.init_db()
     db.utils.execute_sql("""CREATE TABLE IF NOT EXISTS
                                 player_ids (PLAYER_ID text, PLAYER_NAME text, SEASON text);""")