Merge pull request #57 from bellingcat/dev

dev -> 3.3.0.0

Knew Karma/KnewKarma/Utilities/CoreUtils.vb → Knew Karma/KnewKarma/Assets/CoreUtils.vb

@@ -62,33 +62,6 @@ Public Class CoreUtils
         End If
     End Function
 
-    ''' <summary>
-    ''' Asynchronously checks for available updates and optionally displays a message to the user.
-    ''' </summary>
-    ''' <param name="IsAutoCheck">Indicates whether the update check is triggered automatically.</param>
-    ''' <returns>A task representing the asynchronous operation.</returns>
-    Public Shared Async Function AsyncCheckUpdates() As Task
-        AboutWindow.Version.Text = "Checking for Updates..."
-        ' Creating a new instance of the ApiHandler class to interact with the API.
-        Dim Api As New ApiHandler()
-
-        ' Making an asynchronous request to check for updates.
-        Dim data As JObject = Await Api.AsyncGetUpdates()
-
-        ' Checking if data is not null before proceeding with extracting information from it.
-        If data IsNot Nothing Then
-            ' Extracting the tag name, body, and HTML URL from the data.
-            Dim tagName As String = data("tag_name")?.ToString
-
-            ' Checking if the current version is the latest version.
-            If tagName = My.Application.Info.Version.ToString Then
-                AboutWindow.Version.Text = $"Up-to-date: {My.Application.Info.Version}"
-            Else
-                AboutWindow.Version.Text = $"Updates found: {tagName}"
-                AboutWindow.ButtonGetUpdates.Enabled = True
-            End If
-        End If
-    End Function
 
     ''' <summary>
     ''' Checks whether the given JSON data (either JObject or JArray) is null or empty.
@@ -107,11 +80,11 @@ Public Class CoreUtils
     End Function
 
     ''' <summary>
-    ''' Converts a Unix timestamp with possible decimal points to a formatted datetime string.
+    ''' Converts a Unix timestamp with possible decimal points to a formatted datetime.utc string.
     ''' </summary>
     ''' <param name="timestamp">The Unix timestamp to be converted.</param>
     ''' <returns>A formatted datetime string in the format "dd MMMM yyyy, hh:mm:ss.fff tt".</returns>
-    Public Shared Function ConvertTimestampToDatetime(ByVal timestamp As Double) As String
+    Public Shared Function UnixTimestampToUtc(ByVal timestamp As Double) As String
         Dim utcFromTimestamp As Date = New DateTime(
             1970,
             1,

Knew Karma/KnewKarma/Handlers/ApiHandler.vb

@@ -62,8 +62,52 @@ Public Class ApiHandler
     ''' <summary>
     ''' Asynchronously fetches the program's update information from GitHub.
     ''' <summary>
-    Public Async Function AsyncGetUpdates() As Task(Of JObject)
-        Return Await AsyncGetData(endpoint:="https://api.github.com/repos/bellingcat/knewkarma/releases/latest")
+    Public Async Function AsyncGetUpdates() As Task
+        AboutWindow.Version.Text = "Checking for updates..."
+        ' Making an asynchronous request to check for updates.
+        Dim data As JObject = Await AsyncGetData(endpoint:="https://api.github.com/repos/rly0nheart/knewkarma/releases/latest")
+
+        ' Checking if data is not null before proceeding with extracting information from it.
+        If data IsNot Nothing Then
+            ' Extracting the tag name, body, and HTML URL from the data.
+            Dim tagName As String = data("tag_name")?.ToString
+
+            ' Checking if the current version is the latest version.
+            If tagName = My.Application.Info.Version.ToString Then
+                AboutWindow.Version.Text = $"Up-to-date: {My.Application.Info.Version}"
+            Else
+                AboutWindow.Version.Text = $"Updates found: {tagName}"
+                AboutWindow.ButtonGetUpdates.Enabled = True
+            End If
+        End If
+    End Function
+
+
+    ''' <summary>
+    ''' Asynchronously retrieves profile data from a specified source.
+    ''' </summary>
+    ''' <param name="profileType">The type of profile to retrieve.</param>
+    ''' <param name="profileSource">The source from where the profile should be fetched (e.g., specific user or subreddit).</param>
+    ''' <returns>A Task(Of JObject) representing the asynchronous operation, which upon completion returns a Jobject of profile data.</returns>
+    Public Async Function AsyncGetProfile(
+                                         profileType As String, profileSource As String) As Task(Of JObject)
+        Dim profileTypeMap As New List(Of Tuple(Of String, String)) From {
+            Tuple.Create("user_profile", $"{BASE_ENDPOINT}/user/{profileSource}/about.json"),
+            Tuple.Create("subreddit_profile", $"{BASE_ENDPOINT}/r/{profileSource}/about.json")
+        }
+
+        Dim profileEndpoint As String = Nothing
+
+        For Each Type In profileTypeMap
+            If Type.Item1 = profileType Then
+                profileEndpoint = Type.Item2
+                Exit For
+            End If
+        Next
+
+        Dim profile As JObject = Await AsyncGetData(endpoint:=profileEndpoint)
+
+        Return If(profile IsNot Nothing AndAlso profile?("data") IsNot Nothing, profile?("data"), New JObject())
     End Function
 
 
@@ -95,23 +139,13 @@ Public Class ApiHandler
         End If
 
         Dim postsEndpoint As String = postsTypeMap(postsType)
-        Return Await PaginatedPosts(postsEndpoint, postsLimit)
-    End Function
-
-    ''' <summary>
-    ''' Retrieves posts in a paginated manner until the specified limit is reached.
-    ''' </summary>
-    ''' <param name="endpoint">The API endpoint for retrieving posts.</param>
-    ''' <param name="limit">The limit on the number of posts to retrieve.</param>
-    ''' <returns>A Task(Of JArray) representing the asynchronous operation, which upon completion returns a JArray of posts.</returns>
-    Private Async Function PaginatedPosts(endpoint As String, limit As Integer) As Task(Of JArray)
         Dim allPosts As New JArray()
-        Dim lastPostId As String = ""
-        Dim useAfter As Boolean = limit > 100
+        Dim lastPostId As String = String.Empty
+        Dim paginatePosts As Boolean = postsLimit > 100
 
-        While allPosts.Count < limit
-            Dim endpointWithAfter As String = If(useAfter And Not String.IsNullOrEmpty(lastPostId), $"{endpoint}&after={lastPostId}", endpoint)
-            Dim postsData As JObject = Await AsyncGetData(endpoint:=endpointWithAfter)
+        While allPosts.Count < postsLimit
+            Dim paginationEndpoint As String = If(paginatePosts And Not String.IsNullOrEmpty(lastPostId), $"{postsEndpoint}&after={lastPostId}", postsEndpoint)
+            Dim postsData As JObject = Await AsyncGetData(endpoint:=paginationEndpoint)
             Dim postsChildren As JArray = postsData("data")("children")
 
             If postsChildren.Count = 0 Then
@@ -125,32 +159,5 @@ Public Class ApiHandler
 
         Return allPosts
     End Function
-
-
-    ''' <summary>
-    ''' Asynchronously retrieves profile data from a specified source.
-    ''' </summary>
-    ''' <param name="profileType">The type of profile to retrieve.</param>
-    ''' <param name="profileSource">The source from where the profile should be fetched (e.g., specific user or subreddit).</param>
-    ''' <returns>A Task(Of JObject) representing the asynchronous operation, which upon completion returns a Jobject of profile data.</returns>
-    Public Async Function AsyncGetProfile(
-                                         profileType As String, profileSource As String) As Task(Of JObject)
-        Dim profileTypeMap As New List(Of Tuple(Of String, String)) From {
-            Tuple.Create("user_profile", $"{BASE_ENDPOINT}/user/{profileSource}/about.json"),
-            Tuple.Create("subreddit_profile", $"{BASE_ENDPOINT}/r/{profileSource}/about.json")
-        }
-
-        Dim profileEndpoint As String = Nothing
-
-        For Each Type In profileTypeMap
-            If Type.Item1 = profileType Then
-                profileEndpoint = Type.Item2
-                Exit For
-            End If
-        Next
-
-        Dim profile As JObject = Await AsyncGetData(endpoint:=profileEndpoint)
-
-        Return If(profile IsNot Nothing AndAlso profile?("data") IsNot Nothing, profile?("data"), New JObject())
-    End Function
 End Class
+

Knew Karma/KnewKarma/Handlers/DataGridViewHandler.vb

@@ -51,7 +51,7 @@ Public Class DataGridViewer
             ' Special handling for the "created" timestamp
             If RowKey = "created" Then
                 Dim timestamp As Double = CDbl(data("data")(RowKey))
-                value = CoreUtils.ConvertTimestampToDatetime(timestamp)
+                value = CoreUtils.UnixTimestampToUtc(timestamp:=timestamp)
             Else
                 value = If(data("data")(RowKey)?.ToString(), "N/A")
             End If

Knew Karma/KnewKarma/KnewKarma.vbproj

@@ -12,11 +12,11 @@
     <PackageProjectUrl>https://github.com/bellingcat/knewkarma/wiki</PackageProjectUrl>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <RepositoryUrl>https://github.com/bellingcat/knewkarma</RepositoryUrl>
-    <AssemblyVersion>3.2.0.0</AssemblyVersion>
-    <FileVersion>3.2.0.0</FileVersion>
+    <AssemblyVersion>3.3.0.0</AssemblyVersion>
+    <FileVersion>3.3.0.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
-    <Version>3.2.0</Version>
+    <Version>3.3.0</Version>
     <PackageTags>reddit;scraper;reddit-scraper;osint;reddit-data</PackageTags>
     <PackageReleaseNotes></PackageReleaseNotes>
     <AnalysisLevel>6.0-recommended</AnalysisLevel>

Knew Karma/KnewKarma/KnewKarma.vbproj.user

@@ -13,6 +13,9 @@
     <Compile Update="Windows\PostsWindow.vb">
       <SubType>Form</SubType>
     </Compile>
+    <Compile Update="Windows\SubredditProfileWindow.vb">
+      <SubType>Form</SubType>
+    </Compile>
     <Compile Update="Windows\UserProfileWindow.vb">
       <SubType>Form</SubType>
     </Compile>

Knew Karma/KnewKarma/README.md

@@ -1,27 +1,139 @@
-![carbon](https://github.com/bellingcat/knewkarma/assets/74001397/a7165335-89d4-4632-b3af-2f3bb03082bb)
+![knewkarma](https://github.com/bellingcat/knewkarma/assets/74001397/45262d9d-6633-418d-9ace-7c3c88b5ca36)
+
 
 A **Reddit** Data Analysis Toolkit.
 
 [![.Net](https://img.shields.io/badge/Visual%20Basic%20.NET-5C2D91?style=flat&logo=.net&logoColor=white)](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3A%22Visual+Basic+.NET%22&type=code) [![Python](https://img.shields.io/badge/Python-3670A0?style=flat&logo=python&logoColor=ffdd54)](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3APython&type=code) [![Docker](https://img.shields.io/badge/Dockefile-%230db7ed.svg?style=flat&logo=docker&logoColor=white)](https://github.com/search?q=repo%3Abellingcat%2Fknewkarma++language%3ADockerfile&type=code) [![PyPI - Version](https://img.shields.io/pypi/v/knewkarma?style=flat&logo=pypi&logoColor=ffdd54&label=PyPI&labelColor=3670A0&color=3670A0)](https://pypi.org/project/knewkarma)  [![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=flat&logo=buy-me-a-coffee&logoColor=black)](https://buymeacoffee.com/_rly0nheart)
+***
 
 # Feature Overview
 
-* **<ins>Knew Karma can get the following Reddit data from individual targets</ins>**:
+## Knew Karma CLI/GUI
+
+- [x] **<ins>Knew Karma can get the following Reddit data from individual targets</ins>**:
     * **User**: *Profile*, *Posts*, *Comments*
     * **Subreddit**: *Profile*, *Posts*
-    * **Post**: *Data*, *Comments* (available only in the CLI)
-* **<ins>It can also get posts from various sources, such as</ins>**:
+- [x] **<ins>It can also get posts from various sources, such as</ins>**:
     * **Searching**: Allows getting posts that match the user-provided query from all over Reddit
     * **Reddit Front-Page**: Allows getting posts from the Reddit Front-Page
     * **Listing**: Allows getting posts from a user-specified Reddit Listing
-* **<ins>Bonus Features</ins>**
-    * **CLI/GUI**
+- [x] **<ins>Bonus Features</ins>**
+    * **Fully Async (both in the CLI and GUI)**
     * **Dark Mode** (*GUI Automatic/Manual*)
     * **Write data to files** (*JSON/CSV*)
 
+## Knew Karma Python Library
+
+<details>
+    <summary style="text-decoration: underline;">Code Examples</summary>
+
+### Get User Data
+
+```python
+import asyncio
+import aiohttp
+from knewkarma import RedditUser
+
+
+# Define an asynchronous function to fetch User
+async def async_user(username: str, data_timeframe: str, data_limit: int, data_sort: str):
+    # Initialize a RedditUser object with the specified username, data timeframe, limit, and sorting criteria
+    user = RedditUser(username=username, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort)
+
+    # Establish an asynchronous HTTP session
+    async with aiohttp.ClientSession() as session:
+        # Fetch user's profile
+        profile = await user.profile(session=session)
+
+        # Fetch user's posts
+        posts = await user.posts(session=session)
+
+        # Fetch user's comments
+        comments = await user.comments(session=session)
+
+        print(profile)
+        print(posts)
+        print(comments)
+
+
+# Run the asynchronous function with a specified username, data limit, and sorting parameter
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_user(username="automoderator", data_timeframe="year", data_limit=100, data_sort="all"))
+```
+
+### Get Subreddit Data
+
+````python
+import asyncio
+import aiohttp
+from knewkarma import RedditSub
+
+
+async def async_subreddit(subreddit_name: str, data_timeframe: str, data_limit: int, data_sort: str):
+    # Initialize a RedditSub object with the specified subreddit, data timeframe, limit, and sorting criteria
+    subreddit = RedditSub(
+        subreddit=subreddit_name, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort
+    )
+
+    # Create an asynchronous HTTP session
+    async with aiohttp.ClientSession() as session:
+        # Fetch subreddit's profile
+        profile = await subreddit.profile(session=session)
+
+        # Fetch subreddit's posts
+        posts = await subreddit.posts(session=session)
+
+        print(profile)
+        print(posts)
+
+
+# Run the asynchronous function with specified subreddit name, data limit, and sorting criteria
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(
+    async_subreddit(subreddit_name="MachineLearning", data_timeframe="year", data_limit=100, data_sort="top")
+)
+````
+
+### Get Posts
+
+```python
+import asyncio
+import aiohttp
+from knewkarma import RedditPosts
+
+
+async def async_posts(timeframe: str, limit: int, sort: str):
+    # Initialize RedditPosts with the specified timeframe, limit and sorting criteria
+    posts = RedditPosts(timeframe=timeframe, limit=limit, sort=sort)
+
+    # Create an asynchronous HTTP session
+    async with aiohttp.ClientSession() as session:
+        # Fetch front page posts
+        front_page_posts = await posts.front_page(session=session)
+        # Fetch posts from a specified listing ('best')
+        listing_posts = await posts.listing(listings_name="best", session=session)
+        # Fetch posts that match the specified search query 'covid-19'
+        search_results = await posts.search(query="covid-19", session=session)
+
+        print(front_page_posts)
+        print(listing_posts)
+        print(search_results)
+
+
+# Run the asynchronous function with a specified limit and sorting parameter
+# timeframes: ["all", "hour", "day", "month", "year"]
+# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
+asyncio.run(async_posts(timeframe="year", limit=100, sort="all"))
+```
+
+</details>
+
 # Documentation
 
 *[Refer to the Wiki](https://github.com/bellingcat/knewkarma/wiki) for Installation, Usage and Uninstallation
 instructions.*
 ***
 [![me](https://github.com/bellingcat/knewkarma/assets/74001397/efd19c7e-9840-4969-b33c-04087e73e4da)](https://about.me/rly0nheart)
+

Knew Karma/KnewKarma/Windows/AboutWindow.vb

@@ -22,7 +22,8 @@
         'Version.Text = $"Version {My.Application.Info.Version}"
         Copyright.Text = My.Application.Info.Copyright
 
-        Await CoreUtils.AsyncCheckUpdates()
+        Dim ApiHandler As New ApiHandler()
+        Await ApiHandler.AsyncGetUpdates()
     End Sub
 
     ''' <summary>