Skip to content

Commit

Permalink
Merge pull request #52 from bellingcat/dev
Browse files Browse the repository at this point in the history
dev -> 3.1.0.0
  • Loading branch information
rly0nheart committed Dec 1, 2023
2 parents af02b81 + db8db30 commit 10a6a08
Show file tree
Hide file tree
Showing 14 changed files with 178 additions and 87 deletions.
6 changes: 3 additions & 3 deletions Knew Karma/KnewKarma/KnewKarma.vbproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
<PackageProjectUrl>https://github.com/bellingcat/knewkarma/wiki</PackageProjectUrl>
<PackageReadmeFile>README.md</PackageReadmeFile>
<RepositoryUrl>https://github.com/bellingcat/knewkarma</RepositoryUrl>
<AssemblyVersion>3.0.0.0</AssemblyVersion>
<FileVersion>3.0.0.0</FileVersion>
<AssemblyVersion>3.1.0.0</AssemblyVersion>
<FileVersion>3.1.0.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
<Version>3.0.0</Version>
<Version>3.1.0</Version>
<PackageTags>reddit;scraper;reddit-scraper;osint;reddit-data</PackageTags>
<PackageReleaseNotes></PackageReleaseNotes>
<AnalysisLevel>6.0-recommended</AnalysisLevel>
Expand Down
3 changes: 0 additions & 3 deletions Knew Karma/KnewKarma/KnewKarma.vbproj.user
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
<Compile Update="Windows\PostsWindow.vb">
<SubType>Form</SubType>
</Compile>
<Compile Update="Windows\SubredditProfileWindow.vb">
<SubType>Form</SubType>
</Compile>
<Compile Update="Windows\UserProfileWindow.vb">
<SubType>Form</SubType>
</Compile>
Expand Down
6 changes: 3 additions & 3 deletions Knew Karma/KnewKarmaSetup/KnewKarmaSetup.vdproj
Original file line number Diff line number Diff line change
Expand Up @@ -229,15 +229,15 @@
{
"Name" = "8:Microsoft Visual Studio"
"ProductName" = "8:Knew Karma"
"ProductCode" = "8:{29CB9712-39D1-49FD-AAA9-60812E068E52}"
"PackageCode" = "8:{54B3DA36-E5E3-45EC-AB0D-3CB2210F84F8}"
"ProductCode" = "8:{024B1DD3-3E98-4906-A6F4-6E200AD1A66C}"
"PackageCode" = "8:{3765B586-DBC1-4999-B7D6-6CDF9930944A}"
"UpgradeCode" = "8:{9B03AD0F-0C14-4075-AB75-01CD38A594B4}"
"AspNetVersion" = "8:2.0.50727.0"
"RestartWWWService" = "11:FALSE"
"RemovePreviousVersions" = "11:TRUE"
"DetectNewerInstalledVersion" = "11:TRUE"
"InstallAllUsers" = "11:FALSE"
"ProductVersion" = "8:3.0.0"
"ProductVersion" = "8:3.1.0"
"Manufacturer" = "8:Richard Mwewa"
"ARPHELPTELEPHONE" = "8:"
"ARPHELPLINK" = "8:https://github.com/bellingcat/knewkarma/wiki"
Expand Down
34 changes: 19 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
![carbon](https://github.com/bellingcat/knewkarma/assets/74001397/ccc5f890-6c0a-49a4-a6f7-afd398924e7a)
![carbon](https://github.com/bellingcat/knewkarma/assets/74001397/3c1b4238-ea12-4742-974c-14dadd3a280a)

A **Reddit** Data Analysis Toolkit.

Expand Down Expand Up @@ -35,9 +35,9 @@ from knewkarma import RedditUser


# Define an asynchronous function to fetch User
async def async_user(username: str, data_limit: int, data_sort: str):
# Initialize a RedditUser object with the specified username, data limit, and sorting criteria
user = RedditUser(username=username, data_limit=data_limit, data_sort=data_sort)
async def async_user(username: str, data_timeframe: str, data_limit: int, data_sort: str):
# Initialize a RedditUser object with the specified username, data timeframe, limit, and sorting criteria
user = RedditUser(username=username, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort)

# Establish an asynchronous HTTP session
async with aiohttp.ClientSession() as session:
Expand All @@ -56,7 +56,9 @@ async def async_user(username: str, data_limit: int, data_sort: str):


# Run the asynchronous function with a specified username, data limit, and sorting parameter
asyncio.run(async_user(username="automoderator", data_limit=100, data_sort="all"))
# timeframes: ["all", "hour", "day", "month", "year"]
# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
asyncio.run(async_user(username="automoderator", data_timeframe="year", data_limit=100, data_sort="all"))
```

### Get Subreddit Data
Expand All @@ -67,11 +69,10 @@ import aiohttp
from knewkarma import RedditSub


# Define an asynchronous function to fetch Subreddit data
async def async_subreddit(subreddit_name: str, data_limit: int, data_sort: str):
# Initialize a RedditSub object with the specified subreddit, data limit, and sorting criteria
async def async_subreddit(subreddit_name: str, data_timeframe: str, data_limit: int, data_sort: str):
# Initialize a RedditSub object with the specified subreddit, data timeframe, limit, and sorting criteria
subreddit = RedditSub(
subreddit=subreddit_name, data_limit=data_limit, data_sort=data_sort
subreddit=subreddit_name, data_timeframe=data_timeframe, data_limit=data_limit, data_sort=data_sort
)

# Create an asynchronous HTTP session
Expand All @@ -87,8 +88,10 @@ async def async_subreddit(subreddit_name: str, data_limit: int, data_sort: str):


# Run the asynchronous function with specified subreddit name, data limit, and sorting criteria
# timeframes: ["all", "hour", "day", "month", "year"]
# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
asyncio.run(
async_subreddit(subreddit_name="MachineLearning", data_limit=100, data_sort="top")
async_subreddit(subreddit_name="MachineLearning", data_timeframe="year", data_limit=100, data_sort="top")
)
````

Expand All @@ -100,10 +103,9 @@ import aiohttp
from knewkarma import RedditPosts


# Define an asynchronous function to fetch Reddit posts
async def async_posts(limit: int, sort: str):
# Initialize RedditPosts with the specified limit and sorting criteria
posts = RedditPosts(limit=limit, sort=sort)
async def async_posts(timeframe: str, limit: int, sort: str):
# Initialize RedditPosts with the specified timeframe, limit and sorting criteria
posts = RedditPosts(timeframe=timeframe, limit=limit, sort=sort)

# Create an asynchronous HTTP session
async with aiohttp.ClientSession() as session:
Expand All @@ -120,7 +122,9 @@ async def async_posts(limit: int, sort: str):


# Run the asynchronous function with a specified limit and sorting parameter
asyncio.run(async_posts(limit=100, sort="all"))
# timeframes: ["all", "hour", "day", "month", "year"]
# sorting: ["all", "controversial", "new", "top", "best", "hot", "rising"]
asyncio.run(async_posts(timeframe="year", limit=100, sort="all"))
```

</details>
Expand Down
26 changes: 17 additions & 9 deletions knewkarma/_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from datetime import datetime

import aiohttp
import rich
from rich.pretty import pprint

from ._coreutils import log, save_data, pathfinder
from ._parser import create_parser, version
Expand All @@ -22,20 +22,27 @@ async def setup_cli(arguments: argparse.Namespace):
:param arguments: Argparse namespace object containing parsed command-line arguments.
"""
data_limit: int = arguments.limit
data_timeframe: str = arguments.timeframe
data_sorting: str = arguments.limit
data_limit: int = arguments.limit

user = RedditUser(
username=arguments.username if hasattr(arguments, "username") else None,
data_limit=data_limit,
data_timeframe=data_timeframe,
data_sort=data_sorting,
data_limit=data_limit,
)
subreddit = RedditSub(
subreddit=arguments.subreddit if hasattr(arguments, "subreddit") else None,
data_limit=data_limit,
data_timeframe=data_timeframe,
data_sort=data_sorting,
data_limit=data_limit,
)
posts = RedditPosts(
timeframe=data_timeframe,
sort=data_sorting,
limit=data_limit,
)
posts = RedditPosts(limit=data_limit, sort=data_sorting)

# Mapping of command-line commands to their respective functions
function_mapping: dict = {
Expand All @@ -62,18 +69,19 @@ async def setup_cli(arguments: argparse.Namespace):
),
],
}
async with aiohttp.ClientSession() as request_session:
await get_updates(session=request_session)

if arguments.mode in function_mapping:
if arguments.mode in function_mapping:
async with aiohttp.ClientSession() as request_session:
await get_updates(session=request_session)

mode_action = function_mapping.get(arguments.mode)
is_executed: bool = False

for action, function in mode_action:
if getattr(arguments, action, False):
call_function = await function(session=request_session)

rich.print(call_function)
pprint(call_function, expand_all=True)
is_executed = True

pathfinder()
Expand Down
17 changes: 17 additions & 0 deletions knewkarma/_coreutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import json
import logging
import os
from datetime import datetime

from ._parser import create_parser
from .metadata import (
Expand All @@ -15,6 +16,22 @@
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #


def timestamp_to_utc(timestamp: int) -> str:
"""
Converts a UNIX timestamp to a formatted datetime.utc string.
:param timestamp: The UNIX timestamp to be converted.
:return: A formatted datetime.utc string in the format "dd MMMM yyyy, hh:mm:ssAM/PM"
"""
utc_from_timestamp: datetime = datetime.utcfromtimestamp(timestamp)
datetime_string: str = utc_from_timestamp.strftime("%d %B %Y, %I:%M:%S%p")

return datetime_string


# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #


def pathfinder():
"""
Creates file directories in the user's home directory, if they don't already exist.
Expand Down
19 changes: 14 additions & 5 deletions knewkarma/_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
from rich_argparse import RichHelpFormatter

from .metadata import (
version,
description,
epilog,
posts_examples,
user_examples,
subreddit_examples,
operations_description,
version,
)


Expand Down Expand Up @@ -40,9 +40,16 @@ def create_parser() -> argparse.ArgumentParser:
"--limit",
type=int,
default=100,
help="(bulk) data output limit",
help="(bulk) data output limit (default: %(default)s)",
)
parser.add_argument(
"-t",
"--timeframe",
type=str,
default="all",
choices=["all", "hour", "day", "week", "month", "year"],
help="timeframe to get (bulk) data from (default: %(default)s)",
)

parser.add_argument(
"-s",
"--sort",
Expand All @@ -57,23 +64,25 @@ def create_parser() -> argparse.ArgumentParser:
"rising",
"top",
],
help="(bulk) data sort criterion",
help="(bulk) data sort criterion (default: %(default)s)",
)

parser.add_argument(
"-j",
"--json",
metavar="FILENAME",
help="write output to a specified json file",
)
parser.add_argument(
"-c",
"--csv",
metavar="FILENAME",
help="write output to a specified csv file",
)
parser.add_argument(
"-d",
"--debug",
help="([bold][green]dev[/][/]) run knew karma in debug mode.",
help="[bold][green]dev[/][/]: run knew karma in debug mode.",
action="store_true",
)
parser.add_argument(
Expand Down
24 changes: 14 additions & 10 deletions knewkarma/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,19 +109,19 @@ async def get_updates(session: aiohttp.ClientSession):
# Check for differences in version parts
if remote_parts[0] != local_parts[0]:
update_message = (
f"MAJOR update ({remote_version}) available."
f"MAJOR update ({remote_version}) available:"
f" It might introduce significant changes."
)

elif remote_parts[1] != local_parts[1]:
update_message = (
f"MINOR update ({remote_version}) available."
f"MINOR update ({remote_version}) available:"
f" Includes small feature changes/improvements."
)

elif remote_parts[2] != local_parts[2]:
update_message = (
f"PATCH update ({remote_version}) available."
f"PATCH update ({remote_version}) available:"
f" Generally for bug fixes and small tweaks."
)

Expand Down Expand Up @@ -176,8 +176,9 @@ async def get_profile(


async def get_posts(
session: aiohttp.ClientSession,
limit: int,
session: aiohttp.ClientSession,
timeframe: str = Literal["all", "hour", "day", "week", "month", "year"],
sort: str = Literal[
"all",
"controversial",
Expand All @@ -200,19 +201,22 @@ async def get_posts(
"""
Gets a specified number of posts, with a specified sorting criterion, from the specified source.
:param timeframe: Timeframe from which to get posts.
:param session: aiohttp session to use for the request.
:param limit: Maximum number of posts to get.
:param sort: Posts' sort criterion.
:param posts_type: Type of posts to be fetched
:param posts_source: Source from where posts will be fetched.
"""
source_map = {
"user_posts": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/submitted.json?sort={sort}&limit={limit}",
"user_comments": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/comments.json?sort={sort}&limit={limit}",
"subreddit_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}",
"search_posts": f"{BASE_REDDIT_ENDPOINT}/search.json?q={posts_source}&sort={sort}&limit={limit}",
"listing_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}",
"front_page_posts": f"{BASE_REDDIT_ENDPOINT}/.json?sort={sort}&limit={limit}",
"user_posts": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/"
f"submitted.json?sort={sort}&limit={limit}&t={timeframe}",
"user_comments": f"{BASE_REDDIT_ENDPOINT}/user/{posts_source}/"
f"comments.json?sort={sort}&limit={limit}&t={timeframe}",
"subreddit_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}&t={timeframe}",
"search_posts": f"{BASE_REDDIT_ENDPOINT}/search.json?q={posts_source}&sort={sort}&limit={limit}&t={timeframe}",
"listing_posts": f"{BASE_REDDIT_ENDPOINT}/r/{posts_source}.json?sort={sort}&limit={limit}&t={timeframe}",
"front_page_posts": f"{BASE_REDDIT_ENDPOINT}/.json?sort={sort}&limit={limit}&t={timeframe}",
}

endpoint = source_map.get(posts_type, "")
Expand Down
Loading

0 comments on commit 10a6a08

Please sign in to comment.