Merge pull request #52 from bellingcat/dev

dev -> 3.1.0.0

Knew Karma/KnewKarma/KnewKarma.vbproj

@@ -12,11 +12,11 @@
     <PackageProjectUrl>https://github.com/bellingcat/knewkarma/wiki</PackageProjectUrl>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <RepositoryUrl>https://github.com/bellingcat/knewkarma</RepositoryUrl>
-    <AssemblyVersion>3.0.0.0</AssemblyVersion>
-    <FileVersion>3.0.0.0</FileVersion>
+    <AssemblyVersion>3.1.0.0</AssemblyVersion>
+    <FileVersion>3.1.0.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
-    <Version>3.0.0</Version>
+    <Version>3.1.0</Version>
     <PackageTags>reddit;scraper;reddit-scraper;osint;reddit-data</PackageTags>
     <PackageReleaseNotes></PackageReleaseNotes>
     <AnalysisLevel>6.0-recommended</AnalysisLevel>

Knew Karma/KnewKarma/KnewKarma.vbproj.user

@@ -13,9 +13,6 @@
     <Compile Update="Windows\PostsWindow.vb">
       <SubType>Form</SubType>
     </Compile>
-    <Compile Update="Windows\SubredditProfileWindow.vb">
-      <SubType>Form</SubType>
-    </Compile>
     <Compile Update="Windows\UserProfileWindow.vb">
       <SubType>Form</SubType>
     </Compile>

Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj

@@ -229,15 +229,15 @@
         {
         "Name" = "8:Microsoft Visual Studio"
         "ProductName" = "8:Knew Karma"
-        "ProductCode" = "8:{29CB9712-39D1-49FD-AAA9-60812E068E52}"
-        "PackageCode" = "8:{54B3DA36-E5E3-45EC-AB0D-3CB2210F84F8}"
+        "ProductCode" = "8:{024B1DD3-3E98-4906-A6F4-6E200AD1A66C}"
+        "PackageCode" = "8:{3765B586-DBC1-4999-B7D6-6CDF9930944A}"
         "UpgradeCode" = "8:{9B03AD0F-0C14-4075-AB75-01CD38A594B4}"
         "AspNetVersion" = "8:2.0.50727.0"
         "RestartWWWService" = "11:FALSE"
         "RemovePreviousVersions" = "11:TRUE"
         "DetectNewerInstalledVersion" = "11:TRUE"
         "InstallAllUsers" = "11:FALSE"
-        "ProductVersion" = "8:3.0.0"
+        "ProductVersion" = "8:3.1.0"
         "Manufacturer" = "8:Richard Mwewa"
         "ARPHELPTELEPHONE" = "8:"
         "ARPHELPLINK" = "8:https://github.com/bellingcat/knewkarma/wiki"

README.md

@@ -1,4 +1,4 @@
-![carbon](https://github.com/bellingcat/knewkarma/assets/74001397/ccc5f890-6c0a-49a4-a6f7-afd398924e7a)
+![carbon](https://github.com/bellingcat/knewkarma/assets/74001397/3c1b4238-ea12-4742-974c-14dadd3a280a)
 
 A **Reddit** Data Analysis Toolkit.
 
@@ -35,9 +35,9 @@ from knewkarma import RedditUser
 
 
 # Define an asynchronous function to fetch User
-async def async_user(username: str, data_limit: int, data_sort: str):
-    # Initialize a RedditUser object with the specified username, data limit, and sorting criteria
-    user = RedditUser(username=username, data_limit=data_limit, data_sort=data_sort)
+async def async_user(username: str, data_timeframe: str, data_limit: int, data_sort: str):
+    # Initialize a RedditUser object with the specified username, data timeframe, limit, and sorting criteria
+    user = RedditUser(username=username, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort)
 
     # Establish an asynchronous HTTP session
     async with aiohttp.ClientSession() as session:
@@ -56,7 +56,9 @@ async def async_user(username: str, data_limit: int, data_sort: str):
 
 
 # Run the asynchronous function with a specified username, data limit, and sorting parameter
-asyncio.run(async_user(username="automoderator", data_limit=100, data_sort="all"))
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_user(username="automoderator", data_timeframe="year", data_limit=100, data_sort="all"))
 ```
 
 ### Get Subreddit Data
@@ -67,11 +69,10 @@ import aiohttp
 from knewkarma import RedditSub
 
 
-# Define an asynchronous function to fetch Subreddit data
-async def async_subreddit(subreddit_name: str, data_limit: int, data_sort: str):
-    # Initialize a RedditSub object with the specified subreddit, data limit, and sorting criteria
+async def async_subreddit(subreddit_name: str, data_timeframe: str, data_limit: int, data_sort: str):
+    # Initialize a RedditSub object with the specified subreddit, data timeframe, limit, and sorting criteria
     subreddit = RedditSub(
-        subreddit=subreddit_name, data_limit=data_limit, data_sort=data_sort
+        subreddit=subreddit_name, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort
     )
 
     # Create an asynchronous HTTP session
@@ -87,8 +88,10 @@ async def async_subreddit(subreddit_name: str, data_limit: int, data_sort: str):
 
 
 # Run the asynchronous function with specified subreddit name, data limit, and sorting criteria
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
 asyncio.run(
-    async_subreddit(subreddit_name="MachineLearning", data_limit=100, data_sort="top")
+    async_subreddit(subreddit_name="MachineLearning", data_timeframe="year", data_limit=100, data_sort="top")
 )
 ````
 
@@ -100,10 +103,9 @@ import aiohttp
 from knewkarma import RedditPosts
 
 
-# Define an asynchronous function to fetch Reddit posts
-async def async_posts(limit: int, sort: str):
-    # Initialize RedditPosts with the specified limit and sorting criteria
-    posts = RedditPosts(limit=limit, sort=sort)
+async def async_posts(timeframe: str, limit: int, sort: str):
+    # Initialize RedditPosts with the specified timeframe, limit and sorting criteria
+    posts = RedditPosts(timeframe=timeframe, limit=limit, sort=sort)
 
     # Create an asynchronous HTTP session
     async with aiohttp.ClientSession() as session:
@@ -120,7 +122,9 @@ async def async_posts(limit: int, sort: str):
 
 
 # Run the asynchronous function with a specified limit and sorting parameter
-asyncio.run(async_posts(limit=100, sort="all"))
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_posts(timeframe="year", limit=100, sort="all"))
 ```
 
 </details>

knewkarma/_cli.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 
 import aiohttp
-import rich
+from rich.pretty import pprint
 
 from ._coreutils import log, save_data, pathfinder
 from ._parser import create_parser, version
@@ -22,20 +22,27 @@ async def setup_cli(arguments: argparse.Namespace):
 
     :param arguments: Argparse namespace object  containing parsed command-line arguments.
     """
-    data_limit: int = arguments.limit
+    data_timeframe: str = arguments.timeframe
     data_sorting: str = arguments.limit
+    data_limit: int = arguments.limit
 
     user = RedditUser(
         username=arguments.username if hasattr(arguments, "username") else None,
-        data_limit=data_limit,
+        data_timeframe=data_timeframe,
         data_sort=data_sorting,
+        data_limit=data_limit,
     )
     subreddit = RedditSub(
         subreddit=arguments.subreddit if hasattr(arguments, "subreddit") else None,
-        data_limit=data_limit,
+        data_timeframe=data_timeframe,
         data_sort=data_sorting,
+        data_limit=data_limit,
+    )
+    posts = RedditPosts(
+        timeframe=data_timeframe,
+        sort=data_sorting,
+        limit=data_limit,
     )
-    posts = RedditPosts(limit=data_limit, sort=data_sorting)
 
     # Mapping of command-line commands to their respective functions
     function_mapping: dict = {
@@ -62,18 +69,19 @@ async def setup_cli(arguments: argparse.Namespace):
             ),
         ],
     }
-    async with aiohttp.ClientSession() as request_session:
-        await get_updates(session=request_session)
 
-        if arguments.mode in function_mapping:
+    if arguments.mode in function_mapping:
+        async with aiohttp.ClientSession() as request_session:
+            await get_updates(session=request_session)
+
             mode_action = function_mapping.get(arguments.mode)
             is_executed: bool = False
 
             for action, function in mode_action:
                 if getattr(arguments, action, False):
                     call_function = await function(session=request_session)
 
-                    rich.print(call_function)
+                    pprint(call_function, expand_all=True)
                     is_executed = True
 
                     pathfinder()

knewkarma/_coreutils.py

@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+from datetime import datetime
 
 from ._parser import create_parser
 from .metadata import (
@@ -15,6 +16,22 @@
 # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
 
 
+def timestamp_to_utc(timestamp: int) -> str:
+    """
+    Converts a UNIX timestamp to a formatted datetime.utc string.
+
+    :param timestamp: The UNIX timestamp to be converted.
+    :return: A formatted datetime.utc string in the format "dd MMMM yyyy, hh:mm:ssAM/PM"
+    """
+    utc_from_timestamp: datetime = datetime.utcfromtimestamp(timestamp)
+    datetime_string: str = utc_from_timestamp.strftime("%d %B %Y, %I:%M:%S%p")
+
+    return datetime_string
+
+
+# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
+
+
 def pathfinder():
     """
     Creates file directories in the user's home directory, if they don't already exist.

knewkarma/_parser.py

@@ -6,13 +6,13 @@
 from rich_argparse import RichHelpFormatter
 
 from .metadata import (
-    version,
     description,
     epilog,
     posts_examples,
     user_examples,
     subreddit_examples,
     operations_description,
+    version,
 )
 
 
@@ -40,9 +40,16 @@ def create_parser() -> argparse.ArgumentParser:
         "--limit",
         type=int,
         default=100,
-        help="(bulk) data output limit",
+        help="(bulk) data output limit (default: %(default)s)",
+    )
+    parser.add_argument(
+        "-t",
+        "--timeframe",
+        type=str,
+        default="all",
+        choices=["all", "hour", "day", "week", "month", "year"],
+        help="timeframe to get (bulk) data from (default: %(default)s)",
     )
-
     parser.add_argument(
         "-s",
         "--sort",
@@ -57,23 +64,25 @@ def create_parser() -> argparse.ArgumentParser:
             "rising",
             "top",
         ],
-        help="(bulk) data sort criterion",
+        help="(bulk) data sort criterion (default: %(default)s)",
     )
 
     parser.add_argument(
         "-j",
         "--json",
+        metavar="FILENAME",
         help="write output to a specified json file",
     )
     parser.add_argument(
         "-c",
         "--csv",
+        metavar="FILENAME",
         help="write output to a specified csv file",
     )
     parser.add_argument(
         "-d",
         "--debug",
-        help="([bold][green]dev[/][/]) run knew karma in debug mode.",
+        help="[bold][green]dev[/][/]: run knew karma in debug mode.",
         action="store_true",
     )
     parser.add_argument(

knewkarma/api.py

@@ -109,19 +109,19 @@ async def get_updates(session: aiohttp.ClientSession):
         # Check for differences in version parts
         if remote_parts[0] != local_parts[0]:
             update_message = (
-                f"MAJOR update ({remote_version}) available."
+                f"MAJOR update ({remote_version}) available:"
                 f" It might introduce significant changes."
             )
 
         elif remote_parts[1] != local_parts[1]:
             update_message = (
-                f"MINOR update ({remote_version}) available."
+                f"MINOR update ({remote_version}) available:"
                 f" Includes small feature changes/improvements."
             )
 
         elif remote_parts[2] != local_parts[2]:
             update_message = (
-                f"PATCH update ({remote_version}) available."
+                f"PATCH update ({remote_version}) available:"
                 f" Generally for bug fixes and small tweaks."
             )
 
@@ -176,8 +176,9 @@ async def get_profile(
 
 
 async def get_posts(
-    session: aiohttp.ClientSession,
     limit: int,
+    session: aiohttp.ClientSession,
+    timeframe: str = Literal["all", "hour", "day", "week", "month", "year"],
     sort: str = Literal[
         "all",
         "controversial",
@@ -200,19 +201,22 @@ async def get_posts(
     """
     Gets a specified number of posts, with a specified sorting criterion, from the specified source.
 
+    :param timeframe: Timeframe from which to get posts.
     :param session: aiohttp session to use for the request.
     :param limit: Maximum number of posts to get.
     :param sort: Posts' sort criterion.
     :param posts_type: Type of posts to be fetched
     :param posts_source: Source from where posts will be fetched.
     """
     source_map = {
-        "user_posts": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/submitted.json?sort={sort}&limit={limit}",
-        "user_comments": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/comments.json?sort={sort}&limit={limit}",
-        "subreddit_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}",
-        "search_posts": f"{BASE_REDDIT_ENDPOINT}/search.json?q={posts_source}&sort={sort}&limit={limit}",
-        "listing_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}",
-        "front_page_posts": f"{BASE_REDDIT_ENDPOINT}/.json?sort={sort}&limit={limit}",
+        "user_posts": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/"
+        f"submitted.json?sort={sort}&limit={limit}&t={timeframe}",
+        "user_comments": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/"
+        f"comments.json?sort={sort}&limit={limit}&t={timeframe}",
+        "subreddit_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}&t={timeframe}",
+        "search_posts": f"{BASE_REDDIT_ENDPOINT}/search.json?q={posts_source}&sort={sort}&limit={limit}&t={timeframe}",
+        "listing_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}&t={timeframe}",
+        "front_page_posts": f"{BASE_REDDIT_ENDPOINT}/.json?sort={sort}&limit={limit}&t={timeframe}",
     }
 
     endpoint = source_map.get(posts_type, "")