diff --git a/Knew Karma/KnewKarma/Utilities/CoreUtils.vb b/Knew Karma/KnewKarma/Assets/CoreUtils.vb
similarity index 87%
rename from Knew Karma/KnewKarma/Utilities/CoreUtils.vb
rename to Knew Karma/KnewKarma/Assets/CoreUtils.vb
index 8fe7ab1..ede95df 100644
--- a/Knew Karma/KnewKarma/Utilities/CoreUtils.vb
+++ b/Knew Karma/KnewKarma/Assets/CoreUtils.vb
@@ -62,33 +62,6 @@ Public Class CoreUtils
End If
End Function
- '''
- ''' Asynchronously checks for available updates and optionally displays a message to the user.
- '''
- ''' Indicates whether the update check is triggered automatically.
- ''' A task representing the asynchronous operation.
- Public Shared Async Function AsyncCheckUpdates() As Task
- AboutWindow.Version.Text = "Checking for Updates..."
- ' Creating a new instance of the ApiHandler class to interact with the API.
- Dim Api As New ApiHandler()
-
- ' Making an asynchronous request to check for updates.
- Dim data As JObject = Await Api.AsyncGetUpdates()
-
- ' Checking if data is not null before proceeding with extracting information from it.
- If data IsNot Nothing Then
- ' Extracting the tag name, body, and HTML URL from the data.
- Dim tagName As String = data("tag_name")?.ToString
-
- ' Checking if the current version is the latest version.
- If tagName = My.Application.Info.Version.ToString Then
- AboutWindow.Version.Text = $"Up-to-date: {My.Application.Info.Version}"
- Else
- AboutWindow.Version.Text = $"Updates found: {tagName}"
- AboutWindow.ButtonGetUpdates.Enabled = True
- End If
- End If
- End Function
'''
''' Checks whether the given JSON data (either JObject or JArray) is null or empty.
@@ -107,11 +80,11 @@ Public Class CoreUtils
End Function
'''
- ''' Converts a Unix timestamp with possible decimal points to a formatted datetime string.
+ ''' Converts a Unix timestamp with possible decimal points to a formatted datetime.utc string.
'''
''' The Unix timestamp to be converted.
''' A formatted datetime string in the format "dd MMMM yyyy, hh:mm:ss.fff tt".
- Public Shared Function ConvertTimestampToDatetime(ByVal timestamp As Double) As String
+ Public Shared Function UnixTimestampToUtc(ByVal timestamp As Double) As String
Dim utcFromTimestamp As Date = New DateTime(
1970,
1,
diff --git a/Knew Karma/KnewKarma/Utilities/Settings.vb b/Knew Karma/KnewKarma/Assets/Settings.vb
similarity index 100%
rename from Knew Karma/KnewKarma/Utilities/Settings.vb
rename to Knew Karma/KnewKarma/Assets/Settings.vb
diff --git a/Knew Karma/KnewKarma/Handlers/ApiHandler.vb b/Knew Karma/KnewKarma/Handlers/ApiHandler.vb
index 1996b6a..ca37f3b 100644
--- a/Knew Karma/KnewKarma/Handlers/ApiHandler.vb
+++ b/Knew Karma/KnewKarma/Handlers/ApiHandler.vb
@@ -62,8 +62,52 @@ Public Class ApiHandler
'''
''' Asynchronously fetches the program's update information from GitHub.
'''
- Public Async Function AsyncGetUpdates() As Task(Of JObject)
- Return Await AsyncGetData(endpoint:="https://api.github.com/repos/bellingcat/knewkarma/releases/latest")
+ Public Async Function AsyncGetUpdates() As Task
+ AboutWindow.Version.Text = "Checking for updates..."
+ ' Making an asynchronous request to check for updates.
+ Dim data As JObject = Await AsyncGetData(endpoint:="https://api.github.com/repos/rly0nheart/knewkarma/releases/latest")
+
+ ' Checking if data is not null before proceeding with extracting information from it.
+ If data IsNot Nothing Then
+ ' Extracting the tag name, body, and HTML URL from the data.
+ Dim tagName As String = data("tag_name")?.ToString
+
+ ' Checking if the current version is the latest version.
+ If tagName = My.Application.Info.Version.ToString Then
+ AboutWindow.Version.Text = $"Up-to-date: {My.Application.Info.Version}"
+ Else
+ AboutWindow.Version.Text = $"Updates found: {tagName}"
+ AboutWindow.ButtonGetUpdates.Enabled = True
+ End If
+ End If
+ End Function
+
+
+ '''
+ ''' Asynchronously retrieves profile data from a specified source.
+ '''
+ ''' The type of profile to retrieve.
+ ''' The source from where the profile should be fetched (e.g., specific user or subreddit).
+ ''' A Task(Of JObject) representing the asynchronous operation, which upon completion returns a Jobject of profile data.
+ Public Async Function AsyncGetProfile(
+ profileType As String, profileSource As String) As Task(Of JObject)
+ Dim profileTypeMap As New List(Of Tuple(Of String, String)) From {
+ Tuple.Create("user_profile", $"{BASE_ENDPOINT}/user/{profileSource}/about.json"),
+ Tuple.Create("subreddit_profile", $"{BASE_ENDPOINT}/r/{profileSource}/about.json")
+ }
+
+ Dim profileEndpoint As String = Nothing
+
+ For Each Type In profileTypeMap
+ If Type.Item1 = profileType Then
+ profileEndpoint = Type.Item2
+ Exit For
+ End If
+ Next
+
+ Dim profile As JObject = Await AsyncGetData(endpoint:=profileEndpoint)
+
+ Return If(profile IsNot Nothing AndAlso profile?("data") IsNot Nothing, profile?("data"), New JObject())
End Function
@@ -95,23 +139,13 @@ Public Class ApiHandler
End If
Dim postsEndpoint As String = postsTypeMap(postsType)
- Return Await PaginatedPosts(postsEndpoint, postsLimit)
- End Function
-
- '''
- ''' Retrieves posts in a paginated manner until the specified limit is reached.
- '''
- ''' The API endpoint for retrieving posts.
- ''' The limit on the number of posts to retrieve.
- ''' A Task(Of JArray) representing the asynchronous operation, which upon completion returns a JArray of posts.
- Private Async Function PaginatedPosts(endpoint As String, limit As Integer) As Task(Of JArray)
Dim allPosts As New JArray()
- Dim lastPostId As String = ""
- Dim useAfter As Boolean = limit > 100
+ Dim lastPostId As String = String.Empty
+ Dim paginatePosts As Boolean = postsLimit > 100
- While allPosts.Count < limit
- Dim endpointWithAfter As String = If(useAfter And Not String.IsNullOrEmpty(lastPostId), $"{endpoint}&after={lastPostId}", endpoint)
- Dim postsData As JObject = Await AsyncGetData(endpoint:=endpointWithAfter)
+ While allPosts.Count < postsLimit
+ Dim paginationEndpoint As String = If(paginatePosts And Not String.IsNullOrEmpty(lastPostId), $"{postsEndpoint}&after={lastPostId}", postsEndpoint)
+ Dim postsData As JObject = Await AsyncGetData(endpoint:=paginationEndpoint)
Dim postsChildren As JArray = postsData("data")("children")
If postsChildren.Count = 0 Then
@@ -125,32 +159,5 @@ Public Class ApiHandler
Return allPosts
End Function
-
-
- '''
- ''' Asynchronously retrieves profile data from a specified source.
- '''
- ''' The type of profile to retrieve.
- ''' The source from where the profile should be fetched (e.g., specific user or subreddit).
- ''' A Task(Of JObject) representing the asynchronous operation, which upon completion returns a Jobject of profile data.
- Public Async Function AsyncGetProfile(
- profileType As String, profileSource As String) As Task(Of JObject)
- Dim profileTypeMap As New List(Of Tuple(Of String, String)) From {
- Tuple.Create("user_profile", $"{BASE_ENDPOINT}/user/{profileSource}/about.json"),
- Tuple.Create("subreddit_profile", $"{BASE_ENDPOINT}/r/{profileSource}/about.json")
- }
-
- Dim profileEndpoint As String = Nothing
-
- For Each Type In profileTypeMap
- If Type.Item1 = profileType Then
- profileEndpoint = Type.Item2
- Exit For
- End If
- Next
-
- Dim profile As JObject = Await AsyncGetData(endpoint:=profileEndpoint)
-
- Return If(profile IsNot Nothing AndAlso profile?("data") IsNot Nothing, profile?("data"), New JObject())
- End Function
End Class
+
diff --git a/Knew Karma/KnewKarma/Handlers/DataGridViewHandler.vb b/Knew Karma/KnewKarma/Handlers/DataGridViewHandler.vb
index 9cd5255..e8049b0 100644
--- a/Knew Karma/KnewKarma/Handlers/DataGridViewHandler.vb
+++ b/Knew Karma/KnewKarma/Handlers/DataGridViewHandler.vb
@@ -51,7 +51,7 @@ Public Class DataGridViewer
' Special handling for the "created" timestamp
If RowKey = "created" Then
Dim timestamp As Double = CDbl(data("data")(RowKey))
- value = CoreUtils.ConvertTimestampToDatetime(timestamp)
+ value = CoreUtils.UnixTimestampToUtc(timestamp:=timestamp)
Else
value = If(data("data")(RowKey)?.ToString(), "N/A")
End If
diff --git a/Knew Karma/KnewKarma/KnewKarma.vbproj b/Knew Karma/KnewKarma/KnewKarma.vbproj
index ace5c61..0e3fef0 100644
--- a/Knew Karma/KnewKarma/KnewKarma.vbproj
+++ b/Knew Karma/KnewKarma/KnewKarma.vbproj
@@ -12,11 +12,11 @@
https://github.com/bellingcat/knewkarma/wiki
README.md
https://github.com/bellingcat/knewkarma
- 3.2.0.0
- 3.2.0.0
+ 3.3.0.0
+ 3.3.0.0
LICENSE
True
- 3.2.0
+ 3.3.0
reddit;scraper;reddit-scraper;osint;reddit-data
6.0-recommended
diff --git a/Knew Karma/KnewKarma/KnewKarma.vbproj.user b/Knew Karma/KnewKarma/KnewKarma.vbproj.user
index ee29f88..471fe9d 100644
--- a/Knew Karma/KnewKarma/KnewKarma.vbproj.user
+++ b/Knew Karma/KnewKarma/KnewKarma.vbproj.user
@@ -13,6 +13,9 @@
Form
+
+ Form
+
Form
diff --git a/Knew Karma/KnewKarma/README.md b/Knew Karma/KnewKarma/README.md
index e6cf90a..79959a3 100644
--- a/Knew Karma/KnewKarma/README.md
+++ b/Knew Karma/KnewKarma/README.md
@@ -1,27 +1,139 @@
-
+
+
A **Reddit** Data Analysis Toolkit.
[](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3A%22Visual+Basic+.NET%22&type=code) [](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3APython&type=code) [](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3ADockerfile&type=code) [](https://pypi.org/project/knewkarma) [](https://buymeacoffee.com/_rly0nheart)
+***
# Feature Overview
-* **Knew Karma can get the following Reddit data from individual targets**:
+## Knew Karma CLI/GUI
+
+- [x] **Knew Karma can get the following Reddit data from individual targets**:
* **User**: *Profile*, *Posts*, *Comments*
* **Subreddit**: *Profile*, *Posts*
- * **Post**: *Data*, *Comments* (available only in the CLI)
-* **It can also get posts from various sources, such as**:
+- [x] **It can also get posts from various sources, such as**:
* **Searching**: Allows getting posts that match the user-provided query from all over Reddit
* **Reddit Front-Page**: Allows getting posts from the Reddit Front-Page
* **Listing**: Allows getting posts from a user-specified Reddit Listing
-* **Bonus Features**
- * **CLI/GUI**
+- [x] **Bonus Features**
+ * **Fully Async (both in the CLI and GUI)**
* **Dark Mode** (*GUI Automatic/Manual*)
* **Write data to files** (*JSON/CSV*)
+## Knew Karma Python Library
+
+
+ Code Examples
+
+### Get User Data
+
+```python
+import asyncio
+import aiohttp
+from knewkarma import RedditUser
+
+
+# Define an asynchronous function to fetch User
+async def async_user(username: str, data_timeframe: str, data_limit: int, data_sort: str):
+ # Initialize a RedditUser object with the specified username, data timeframe, limit, and sorting criteria
+ user = RedditUser(username=username, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort)
+
+ # Establish an asynchronous HTTP session
+ async with aiohttp.ClientSession() as session:
+ # Fetch user's profile
+ profile = await user.profile(session=session)
+
+ # Fetch user's posts
+ posts = await user.posts(session=session)
+
+ # Fetch user's comments
+ comments = await user.comments(session=session)
+
+ print(profile)
+ print(posts)
+ print(comments)
+
+
+# Run the asynchronous function with a specified username, data limit, and sorting parameter
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_user(username="automoderator", data_timeframe="year", data_limit=100, data_sort="all"))
+```
+
+### Get Subreddit Data
+
+````python
+import asyncio
+import aiohttp
+from knewkarma import RedditSub
+
+
+async def async_subreddit(subreddit_name: str, data_timeframe: str, data_limit: int, data_sort: str):
+ # Initialize a RedditSub object with the specified subreddit, data timeframe, limit, and sorting criteria
+ subreddit = RedditSub(
+ subreddit=subreddit_name, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort
+ )
+
+ # Create an asynchronous HTTP session
+ async with aiohttp.ClientSession() as session:
+ # Fetch subreddit's profile
+ profile = await subreddit.profile(session=session)
+
+ # Fetch subreddit's posts
+ posts = await subreddit.posts(session=session)
+
+ print(profile)
+ print(posts)
+
+
+# Run the asynchronous function with specified subreddit name, data limit, and sorting criteria
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(
+ async_subreddit(subreddit_name="MachineLearning", data_timeframe="year", data_limit=100, data_sort="top")
+)
+````
+
+### Get Posts
+
+```python
+import asyncio
+import aiohttp
+from knewkarma import RedditPosts
+
+
+async def async_posts(timeframe: str, limit: int, sort: str):
+ # Initialize RedditPosts with the specified timeframe, limit and sorting criteria
+ posts = RedditPosts(timeframe=timeframe, limit=limit, sort=sort)
+
+ # Create an asynchronous HTTP session
+ async with aiohttp.ClientSession() as session:
+ # Fetch front page posts
+ front_page_posts = await posts.front_page(session=session)
+ # Fetch posts from a specified listing ('best')
+ listing_posts = await posts.listing(listings_name="best", session=session)
+ # Fetch posts that match the specified search query 'covid-19'
+ search_results = await posts.search(query="covid-19", session=session)
+
+ print(front_page_posts)
+ print(listing_posts)
+ print(search_results)
+
+
+# Run the asynchronous function with a specified limit and sorting parameter
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_posts(timeframe="year", limit=100, sort="all"))
+```
+
+
+
# Documentation
*[Refer to the Wiki](https://github.com/bellingcat/knewkarma/wiki) for Installation, Usage and Uninstallation
instructions.*
***
[](https://about.me/rly0nheart)
+
diff --git a/Knew Karma/KnewKarma/Windows/AboutWindow.vb b/Knew Karma/KnewKarma/Windows/AboutWindow.vb
index b9cc9d0..906c745 100644
--- a/Knew Karma/KnewKarma/Windows/AboutWindow.vb
+++ b/Knew Karma/KnewKarma/Windows/AboutWindow.vb
@@ -22,7 +22,8 @@
'Version.Text = $"Version {My.Application.Info.Version}"
Copyright.Text = My.Application.Info.Copyright
- Await CoreUtils.AsyncCheckUpdates()
+ Dim ApiHandler As New ApiHandler()
+ Await ApiHandler.AsyncGetUpdates()
End Sub
'''
diff --git a/Knew Karma/KnewKarma/Windows/MainWindow.Designer.vb b/Knew Karma/KnewKarma/Windows/MainWindow.Designer.vb
index dc7a345..aaa5463 100644
--- a/Knew Karma/KnewKarma/Windows/MainWindow.Designer.vb
+++ b/Knew Karma/KnewKarma/Windows/MainWindow.Designer.vb
@@ -23,7 +23,7 @@ Partial Class MainWindow
Private Sub InitializeComponent()
components = New ComponentModel.Container()
- Dim resources As ComponentModel.ComponentResourceManager = New ComponentModel.ComponentResourceManager(GetType(MainWindow))
+ Dim resources As System.ComponentModel.ComponentResourceManager = New System.ComponentModel.ComponentResourceManager(GetType(MainWindow))
Dim TreeNode1 As TreeNode = New TreeNode("Subreddit")
Dim TreeNode2 As TreeNode = New TreeNode("User")
Dim TreeNode3 As TreeNode = New TreeNode("Front Page")
@@ -495,7 +495,7 @@ Partial Class MainWindow
NumericUpDownSearchResultLimit.Font = New Font("Segoe UI Variable Display Semib", 8.25F, FontStyle.Bold, GraphicsUnit.Point)
NumericUpDownSearchResultLimit.Location = New Point(130, 41)
NumericUpDownSearchResultLimit.Maximum = New Decimal(New Integer() {10000, 0, 0, 0})
- NumericUpDownSearchResultLimit.Minimum = New Decimal(New Integer() {5, 0, 0, 0})
+ NumericUpDownSearchResultLimit.Minimum = New Decimal(New Integer() {100, 0, 0, 0})
NumericUpDownSearchResultLimit.Name = "NumericUpDownSearchResultLimit"
NumericUpDownSearchResultLimit.ReadOnly = True
NumericUpDownSearchResultLimit.Size = New Size(79, 22)
@@ -541,7 +541,7 @@ Partial Class MainWindow
NumericUpDownFrontPageDataLimit.Font = New Font("Segoe UI Variable Text Semibold", 8.25F, FontStyle.Bold, GraphicsUnit.Point)
NumericUpDownFrontPageDataLimit.Location = New Point(130, 41)
NumericUpDownFrontPageDataLimit.Maximum = New Decimal(New Integer() {10000, 0, 0, 0})
- NumericUpDownFrontPageDataLimit.Minimum = New Decimal(New Integer() {5, 0, 0, 0})
+ NumericUpDownFrontPageDataLimit.Minimum = New Decimal(New Integer() {100, 0, 0, 0})
NumericUpDownFrontPageDataLimit.Name = "NumericUpDownFrontPageDataLimit"
NumericUpDownFrontPageDataLimit.ReadOnly = True
NumericUpDownFrontPageDataLimit.Size = New Size(79, 22)
@@ -600,7 +600,7 @@ Partial Class MainWindow
NumericUpDownSubredditPostsLimit.Font = New Font("Segoe UI Variable Display Semib", 8.25F, FontStyle.Bold, GraphicsUnit.Point)
NumericUpDownSubredditPostsLimit.Location = New Point(130, 41)
NumericUpDownSubredditPostsLimit.Maximum = New Decimal(New Integer() {10000, 0, 0, 0})
- NumericUpDownSubredditPostsLimit.Minimum = New Decimal(New Integer() {5, 0, 0, 0})
+ NumericUpDownSubredditPostsLimit.Minimum = New Decimal(New Integer() {100, 0, 0, 0})
NumericUpDownSubredditPostsLimit.Name = "NumericUpDownSubredditPostsLimit"
NumericUpDownSubredditPostsLimit.ReadOnly = True
NumericUpDownSubredditPostsLimit.Size = New Size(79, 22)
@@ -675,7 +675,7 @@ Partial Class MainWindow
NumericUpDownUserDataLimit.Font = New Font("Segoe UI Variable Display Semib", 8.25F, FontStyle.Bold, GraphicsUnit.Point)
NumericUpDownUserDataLimit.Location = New Point(130, 41)
NumericUpDownUserDataLimit.Maximum = New Decimal(New Integer() {10000, 0, 0, 0})
- NumericUpDownUserDataLimit.Minimum = New Decimal(New Integer() {5, 0, 0, 0})
+ NumericUpDownUserDataLimit.Minimum = New Decimal(New Integer() {100, 0, 0, 0})
NumericUpDownUserDataLimit.Name = "NumericUpDownUserDataLimit"
NumericUpDownUserDataLimit.ReadOnly = True
NumericUpDownUserDataLimit.Size = New Size(79, 22)
@@ -927,7 +927,7 @@ Partial Class MainWindow
NumericUpDownPostListingsLimit.Font = New Font("Segoe UI Variable Display Semib", 8.25F, FontStyle.Bold, GraphicsUnit.Point)
NumericUpDownPostListingsLimit.Location = New Point(130, 41)
NumericUpDownPostListingsLimit.Maximum = New Decimal(New Integer() {10000, 0, 0, 0})
- NumericUpDownPostListingsLimit.Minimum = New Decimal(New Integer() {5, 0, 0, 0})
+ NumericUpDownPostListingsLimit.Minimum = New Decimal(New Integer() {100, 0, 0, 0})
NumericUpDownPostListingsLimit.Name = "NumericUpDownPostListingsLimit"
NumericUpDownPostListingsLimit.ReadOnly = True
NumericUpDownPostListingsLimit.Size = New Size(79, 22)
diff --git a/Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj b/Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj
index c1eeb53..4d62f4c 100644
--- a/Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj
+++ b/Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj
@@ -229,15 +229,15 @@
{
"Name" = "8:Microsoft Visual Studio"
"ProductName" = "8:Knew Karma"
- "ProductCode" = "8:{73004786-0578-4532-B92C-A9D8A8B5E8B2}"
- "PackageCode" = "8:{9ECB32FA-4D0C-4D29-9723-17958851D59F}"
+ "ProductCode" = "8:{EC317D6E-D0D6-4C8B-9120-59D6F8AE1626}"
+ "PackageCode" = "8:{5A52E59B-8C23-4400-8D0F-851C28E3DADA}"
"UpgradeCode" = "8:{9B03AD0F-0C14-4075-AB75-01CD38A594B4}"
"AspNetVersion" = "8:2.0.50727.0"
"RestartWWWService" = "11:FALSE"
"RemovePreviousVersions" = "11:TRUE"
"DetectNewerInstalledVersion" = "11:TRUE"
"InstallAllUsers" = "11:FALSE"
- "ProductVersion" = "8:3.2.0"
+ "ProductVersion" = "8:3.3.0"
"Manufacturer" = "8:Richard Mwewa"
"ARPHELPTELEPHONE" = "8:"
"ARPHELPLINK" = "8:https://github.com/bellingcat/knewkarma/wiki"
@@ -777,7 +777,7 @@
{
"{5259A561-127C-4D43-A0A1-72F10C7B3BF8}:_6E15D9F422094BD9809550AF1BA1C161"
{
- "SourcePath" = "8:..\\KnewKarma\\obj\\Release\\net6.0-windows\\apphost.exe"
+ "SourcePath" = "8:..\\KnewKarma\\obj\\Debug\\net6.0-windows\\apphost.exe"
"TargetName" = "8:"
"Tag" = "8:"
"Folder" = "8:_C0F76EDD899B4FFF80C2AC1B5526BC22"
diff --git a/knewkarma/_cli.py b/knewkarma/_cli.py
index 48457ac..bca61a3 100644
--- a/knewkarma/_cli.py
+++ b/knewkarma/_cli.py
@@ -113,6 +113,8 @@ async def setup_cli(arguments: argparse.Namespace):
save_csv=arguments.csv,
)
+ break
+
# -------------------------------------------------------------------- #
if not is_executed:
@@ -124,7 +126,7 @@ async def setup_cli(arguments: argparse.Namespace):
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
-def execute_cli():
+def execute():
"""Main entrypoint for the Knew Karma command-line interface."""
# -------------------------------------------------------------------- #
diff --git a/knewkarma/_coreutils.py b/knewkarma/_coreutils.py
index 08bae58..e448eee 100644
--- a/knewkarma/_coreutils.py
+++ b/knewkarma/_coreutils.py
@@ -4,6 +4,7 @@
import json
import logging
import os
+from datetime import datetime
from typing import Union, List
from ._parser import create_parser
@@ -13,15 +14,13 @@
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
-def timestamp_to_utc(timestamp: int) -> str:
+def unix_timestamp_to_utc(timestamp: int) -> str:
"""
Converts a UNIX timestamp to a formatted datetime.utc string.
:param timestamp: The UNIX timestamp to be converted.
:return: A formatted datetime.utc string in the format "dd MMMM yyyy, hh:mm:ssAM/PM"
"""
- from datetime import datetime
-
utc_from_timestamp: datetime = datetime.utcfromtimestamp(timestamp)
datetime_string: str = utc_from_timestamp.strftime("%d %B %Y, %I:%M:%S%p")
@@ -31,6 +30,30 @@ def timestamp_to_utc(timestamp: int) -> str:
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
+def filename_timestamp() -> str:
+ """
+ Generates a timestamp string suitable for file naming, based on the current date and time.
+ The format of the timestamp is adapted based on the operating system.
+
+ :return: The formatted timestamp as a string. The format is "%d-%B-%Y-%I-%M-%S%p" for Windows
+ and "%d-%B-%Y-%I:%M:%S%p" for non-Windows systems.
+
+ Example
+ -------
+ - Windows: "20-July-1969-08-17-45PM"
+ - Non-Windows: "20-July-1969-08:17:45PM" (format may vary based on the current date and time)
+ """
+ now = datetime.now()
+ return (
+ now.strftime("%d-%B-%Y-%I-%M-%S%p")
+ if os.name == "nt"
+ else now.strftime("%d-%B-%Y-%I:%M:%S%p")
+ )
+
+
+# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
+
+
def pathfinder(directories: list[str]):
"""
Creates directories in knewkarma-data directory of the user's home folder.
@@ -47,8 +70,8 @@ def pathfinder(directories: list[str]):
def save_data(
data: Union[User, Subreddit, List[Union[Post, Comment]]],
save_to_dir: str,
- save_json: bool = False,
- save_csv: bool = False,
+ save_json: Union[bool, str] = False,
+ save_csv: Union[bool, str] = False,
):
"""
Save the given (Reddit) data to a JSON/CSV file based on the save_csv and save_json parameters.
@@ -74,7 +97,9 @@ def save_data(
# -------------------------------------------------------------------- #
if save_json:
- json_path = os.path.join(save_to_dir, "json", f"{save_json}.json")
+ json_path = os.path.join(
+ save_to_dir, "json", f"{save_json.upper()}-{filename_timestamp()}.json"
+ )
with open(json_path, "w", encoding="utf-8") as json_file:
json.dump(function_data, json_file, indent=4)
log.info(
@@ -84,7 +109,9 @@ def save_data(
# -------------------------------------------------------------------- #
if save_csv:
- csv_path = os.path.join(save_to_dir, "csv", f"{save_csv}.csv")
+ csv_path = os.path.join(
+ save_to_dir, "csv", f"{save_csv.upper()}-{filename_timestamp()}.csv"
+ )
with open(csv_path, "w", newline="", encoding="utf-8") as csv_file:
writer = csv.writer(csv_file)
if isinstance(function_data, dict):
diff --git a/knewkarma/_project.py b/knewkarma/_project.py
index d0be520..775e814 100644
--- a/knewkarma/_project.py
+++ b/knewkarma/_project.py
@@ -6,7 +6,7 @@
author: str = "Richard Mwewa"
about_author: str = "https://about.me/rly0nheart"
-version: str = "3.2.0.0"
+version: str = "3.3.0.0"
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
diff --git a/knewkarma/api.py b/knewkarma/api.py
index a833722..888273e 100644
--- a/knewkarma/api.py
+++ b/knewkarma/api.py
@@ -10,7 +10,9 @@
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
BASE_REDDIT_ENDPOINT: str = "https://www.reddit.com"
-PYPI_PROJECT_ENDPOINT: str = "https://pypi.org/pypi/knewkarma/json"
+GITHUB_RELEASE_ENDPOINT: str = (
+ "https://api.github.com/repos/rly0nheart/knewkarma/releases/latest"
+)
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
@@ -91,13 +93,18 @@ async def get_updates(session: aiohttp.ClientSession):
:param session: aiohttp session to use for the request.
"""
+ import rich
+ from rich.markdown import Markdown
# Make a GET request to PyPI to get the project's latest release.
- response: dict = await get_data(endpoint=PYPI_PROJECT_ENDPOINT, session=session)
- release: dict = process_response(response_data=response.get("info", {}))
+ response: dict = await get_data(endpoint=GITHUB_RELEASE_ENDPOINT, session=session)
+ release: dict = process_response(response_data=response, valid_key="tag_name")
if release:
- remote_version: str = release.get("version")
+ remote_version: str = release.get("tag_name")
+ markup_release_notes: str = release.get("body")
+ markdown_release_notes = Markdown(markup=markup_release_notes)
+
# Splitting the version strings into components
remote_parts: list = remote_version.split(".")
local_parts: list = version.split(".")
@@ -109,7 +116,7 @@ async def get_updates(session: aiohttp.ClientSession):
# Check for differences in version parts
if remote_parts[0] != local_parts[0]:
update_message = (
- f"MAJOR update ({remote_version}) available:"
+ f"[bold][red]MAJOR[/][/] update ({remote_version}) available:"
f" It might introduce significant changes."
)
@@ -117,7 +124,7 @@ async def get_updates(session: aiohttp.ClientSession):
elif remote_parts[1] != local_parts[1]:
update_message = (
- f"MINOR update ({remote_version}) available:"
+ f"[bold][blue]MINOR[/][/] update ({remote_version}) available:"
f" Includes small feature changes/improvements."
)
@@ -125,7 +132,7 @@ async def get_updates(session: aiohttp.ClientSession):
elif remote_parts[2] != local_parts[2]:
update_message = (
- f"PATCH update ({remote_version}) available:"
+ f"[bold][green]PATCH[/][/] update ({remote_version}) available:"
f" Generally for bug fixes and small tweaks."
)
@@ -137,7 +144,7 @@ async def get_updates(session: aiohttp.ClientSession):
and remote_parts[3] != local_parts[3]
):
update_message = (
- f"BUILD update ({remote_version}) available."
+ f"[bold][cyan]BUILD[/][/] update ({remote_version}) available."
f" Might be for specific builds or special versions."
)
@@ -145,6 +152,7 @@ async def get_updates(session: aiohttp.ClientSession):
if update_message:
log.info(update_message)
+ rich.print(markdown_release_notes)
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
@@ -227,51 +235,32 @@ async def get_posts(
"front_page_posts": f"{BASE_REDDIT_ENDPOINT}/.json?sort={sort}&limit={limit}&t={timeframe}",
}
+ # ---------------------------------------------------------- #
+
endpoint = source_map.get(posts_type, "")
if not endpoint:
raise ValueError(f"Invalid profile type in {source_map}: {posts_type}")
- all_posts = await paginated_posts(
- posts_endpoint=endpoint, limit=limit, session=session
- )
-
- return all_posts[:limit]
-
-
-# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #
-
-
-async def paginated_posts(
- posts_endpoint: str, limit: int, session: aiohttp.ClientSession
-) -> list:
- """
- Paginates and retrieves posts until the specified limit is reached.
-
- :param posts_endpoint: API endpoint for retrieving posts.
- :param limit: Limit of the number of posts to retrieve.
- :param session: aiohttp session to use for the request.
- :return: A list of all posts.
- """
all_posts: list = []
last_post_id: str = ""
# Determine whether to use the 'after' parameter
- use_after: bool = limit > 100
+ paginate_posts: bool = limit > 100
- while len(all_posts) < limit:
- # ---------------------------------------------------------- #
+ # ---------------------------------------------------------- #
+ while len(all_posts) < limit:
# Make the API request with the 'after' parameter if it's provided and the limit is more than 100
- if use_after and last_post_id:
- endpoint_with_after: str = f"{posts_endpoint}&after={last_post_id}"
+ if paginate_posts and last_post_id:
+ pagination_endpoint: str = f"{endpoint}&after={last_post_id}"
else:
- endpoint_with_after: str = posts_endpoint
+ pagination_endpoint: str = endpoint
# ---------------------------------------------------------- #
raw_posts_data: dict = await get_data(
- endpoint=endpoint_with_after, session=session
+ endpoint=pagination_endpoint, session=session
)
posts_list: list = raw_posts_data.get("data", {}).get("children", [])
diff --git a/knewkarma/base.py b/knewkarma/base.py
index 22249ef..76f5b14 100644
--- a/knewkarma/base.py
+++ b/knewkarma/base.py
@@ -4,7 +4,7 @@
import aiohttp
-from ._coreutils import timestamp_to_utc
+from ._coreutils import unix_timestamp_to_utc
from .api import get_profile, get_posts
from .data import User, Subreddit, Comment, Post
@@ -67,7 +67,7 @@ async def profile(self, session: aiohttp.ClientSession) -> User:
awardee_karma=user_profile.get("awardee_karma"),
total_karma=user_profile.get("total_karma"),
subreddit=user_profile.get("subreddit"),
- created_at=timestamp_to_utc(timestamp=user_profile.get("created")),
+ created_at=unix_timestamp_to_utc(timestamp=user_profile.get("created")),
raw_data=user_profile,
)
@@ -127,7 +127,7 @@ async def comments(self, session: aiohttp.ClientSession) -> List[Comment]:
is_stickied=comment_data.get("stickied"),
is_locked=comment_data.get("locked"),
is_archived=comment_data.get("archived"),
- created_at=timestamp_to_utc(timestamp=comment_data.get("created")),
+ created_at=unix_timestamp_to_utc(timestamp=comment_data.get("created")),
subreddit=comment_data.get("subreddit_name_prefixed"),
subreddit_type=comment_data.get("subreddit_type"),
post_id=comment_data.get("link_id"),
@@ -191,7 +191,9 @@ async def profile(self, session: aiohttp.ClientSession) -> Subreddit:
current_active_users=subreddit_profile.get("accounts_active"),
is_nsfw=subreddit_profile.get("over18"),
language=subreddit_profile.get("lang"),
- created_at=timestamp_to_utc(timestamp=subreddit_profile.get("created")),
+ created_at=unix_timestamp_to_utc(
+ timestamp=subreddit_profile.get("created")
+ ),
raw_data=subreddit_profile,
)
@@ -273,7 +275,7 @@ def process_posts(raw_posts: list) -> List[Post]:
permalink=post_data.get("permalink"),
is_locked=post_data.get("locked"),
is_archived=post_data.get("archived"),
- created_at=timestamp_to_utc(timestamp=post_data.get("created")),
+ created_at=unix_timestamp_to_utc(timestamp=post_data.get("created")),
raw_post=post_data,
)
posts_list.append(post)
diff --git a/pyproject.toml b/pyproject.toml
index 801b23d..26ec542 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "knewkarma"
-version = "3.2.0.0"
+version = "3.3.0.0"
description = "A Reddit Data Analysis Toolkit."
authors = ["Richard Mwewa "]
readme = "README.md"
@@ -31,4 +31,4 @@ rich = "*"
rich-argparse = "*"
[tool.poetry.scripts]
-knewkarma = "knewkarma._cli:execute_cli"
+knewkarma = "knewkarma._cli:execute"