From 0dcd52730faa1a9f49697cb1823d0284424987c4 Mon Sep 17 00:00:00 2001 From: CoolCoderCarl <123qwekir2wl@gmail.com> Date: Tue, 25 Oct 2022 23:14:04 +0300 Subject: [PATCH 1/3] Update README.md & dynaconfig.py & truth_seeker.py --- README.md | 4 +++- app/truth_seeker.py | 21 ++++++++++++++------- dynaconfig.py | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 602a3e3..e98a475 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# truth_seeker \ No newline at end of file +# truth_seeker + +https://newsapi.org/ \ No newline at end of file diff --git a/app/truth_seeker.py b/app/truth_seeker.py index d050d73..9b946a8 100644 --- a/app/truth_seeker.py +++ b/app/truth_seeker.py @@ -1,15 +1,17 @@ import logging +from datetime import date, timedelta from newsapi import NewsApiClient import dynaconfig -# https://newsapi.org/ - -API_KEY = dynaconfig.config["API_KEY"] +API_KEY = dynaconfig.settings["API_KEY"] newsapi = NewsApiClient(api_key=API_KEY) +YESTERDAY = date.today() - timedelta(days=1) +QUERY = dynaconfig.settings["QUERY"] + # Logging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO @@ -22,10 +24,12 @@ ) -def fetch_info(query: str) -> dict: +def fetch_info() -> dict: try: - result = newsapi.get_everything(q=query, sort_by="popularity") - logging.info(f"Searching for {query}") + result = newsapi.get_everything( + q=QUERY, sort_by="popularity", from_param=YESTERDAY + ) + logging.info(f"Searching for {QUERY}") return result except ConnectionError as con_err: logging.error(con_err) @@ -51,4 +55,7 @@ def list_info(fetch_info: dict): if __name__ == "__main__": - list_info(fetch_info("test")) + if dynaconfig.settings["LOAD_TO_DB"]: + pass + else: + list_info(fetch_info()) diff --git a/dynaconfig.py b/dynaconfig.py index 9977d2a..75a0bf9 100644 --- a/dynaconfig.py +++ b/dynaconfig.py @@ -1,5 +1,5 @@ from dynaconf import Dynaconf -config = Dynaconf( +settings = Dynaconf( settings_files=[".secrets.toml"], ) From 127a6f8a8b39b701a8febaedccf6c547786da212 Mon Sep 17 00:00:00 2001 From: CoolCoderCarl <123qwekir2wl@gmail.com> Date: Wed, 26 Oct 2022 19:17:47 +0300 Subject: [PATCH 2/3] Add func to load to db as sqlite --- app/truth_seeker.py | 39 +++++++++++++++++------ databaseconnection.py | 73 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 9 deletions(-) create mode 100644 databaseconnection.py diff --git a/app/truth_seeker.py b/app/truth_seeker.py index 9b946a8..2ad661d 100644 --- a/app/truth_seeker.py +++ b/app/truth_seeker.py @@ -3,14 +3,16 @@ from newsapi import NewsApiClient +import databaseconnection import dynaconfig API_KEY = dynaconfig.settings["API_KEY"] +QUERY = dynaconfig.settings["QUERY"] -newsapi = NewsApiClient(api_key=API_KEY) - +# Use to get news from yesterday to current time YESTERDAY = date.today() - timedelta(days=1) -QUERY = dynaconfig.settings["QUERY"] + +newsapi = NewsApiClient(api_key=API_KEY) # Logging logging.basicConfig( @@ -42,20 +44,39 @@ def fetch_info() -> dict: def list_info(fetch_info: dict): if fetch_info: for article in fetch_info.get("articles"): - for article_point, article_data in article.items(): - if article_point == "urlToImage": + for article_key, article_data in article.items(): + if article_key == "urlToImage": continue - elif article_point == "source": - print(article_point.capitalize(), article_data.get("name")) + elif article_key == "source": + print(article_key.capitalize(), article_data.get("name")) else: - print(article_point.capitalize(), article_data) + print(article_key.capitalize(), article_data) else: logging.warning("Empty response from API") raise IndexError +def load_to_db(): + for article in fetch_info().get("articles"): + article_list = [] + for article_key, article_data in article.items(): + if article_key == "source": + pass + elif article_key == "content": + pass + elif article_key == "urlToImage": + pass + else: + article_list.append(article_data) + article_list.append("True") + databaseconnection.insert_into( + databaseconnection.create_connection(databaseconnection.DB_FILE), + tuple(article_list), + ) + + if __name__ == "__main__": if dynaconfig.settings["LOAD_TO_DB"]: - pass + load_to_db() else: list_info(fetch_info()) diff --git a/databaseconnection.py b/databaseconnection.py new file mode 100644 index 0000000..242fc60 --- /dev/null +++ b/databaseconnection.py @@ -0,0 +1,73 @@ +import logging +import sqlite3 +from pathlib import Path +from sqlite3 import Error + +DB_FILE = Path("test.db") + +CREATE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS news ( +news_id INTEGER PRIMARY KEY, +author TEXT, +title TEXT, +description TEXT, +url TEXT, +pub_date TEXT, +used_bool TEXT +); +""" + +INSERT_INTO_SQL = """ +INSERT INTO news +(author,title,description,url,pub_date,used_bool) +VALUES(?,?,?,?,?,?) +""" + + +# Logging +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO +) +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.ERROR +) + + +def create_connection(db_file: Path): + conn = None + try: + conn = sqlite3.connect(db_file) + logging.info("Database created successfully !") + return conn + except Error as create_conn_err: + logging.error(create_conn_err) + return conn + + +def create_table(conn, create_table_query): + try: + c = conn.cursor() + c.execute(create_table_query) + logging.info("Table created successfully !") + except Error as create_table_err: + logging.error(create_table_err) + + +def insert_into(conn, data: tuple): + try: + cur = conn.cursor() + cur.execute(INSERT_INTO_SQL, data) + conn.commit() + + logging.info("Data inserted successfully !") + return cur.lastrowid + except Error as insert_err: + logging.error(insert_err) + + +if __name__ == "__main__": + conn = create_connection(DB_FILE) + if conn is not None: + create_table(conn, CREATE_TABLE_SQL) + else: + logging.error("Error! Cannot create the database connection.") From 502c7a3330c991aacd3faea9c7797abbf263c1fc Mon Sep 17 00:00:00 2001 From: CoolCoderCarl <123qwekir2wl@gmail.com> Date: Wed, 26 Oct 2022 19:31:27 +0300 Subject: [PATCH 3/3] Add try except block --- app/truth_seeker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/truth_seeker.py b/app/truth_seeker.py index 2ad661d..17c4327 100644 --- a/app/truth_seeker.py +++ b/app/truth_seeker.py @@ -77,6 +77,9 @@ def load_to_db(): if __name__ == "__main__": if dynaconfig.settings["LOAD_TO_DB"]: - load_to_db() + try: + load_to_db() + except BaseException as base_err: + logging.error(base_err) else: list_info(fetch_info())