diff --git a/README.md b/README.md index 21ec2b2..0004bb1 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,8 @@ A graphical interface that displays information in the elevator lobby of Computer Science House. All information displayed has been authorized to been shown. -Documentation for the project can be found be appended /docs to the url -All HTML requests that are sent in the project can be seen by appending /swag - This project uses Python, [FastAPI](https://fastapi.tiangolo.com/), HTML/CSS, and Javascript. -See it live [here](http://jumpstart-cubed.cs.house/)! +See it live [here](http://jumpstart-squared.cs.house/)! ## Installing 1. Clone and cd into the repo: git clone https://github.com/WeatherGod3218/jumpstartV2 @@ -24,10 +21,7 @@ See it live [here](http://jumpstart-cubed.cs.house/)! 2. Copy the .env.template file, rename it to .env and place it in the root folder 3. Ask an RTP for jumpstart secrets, add them to the .env accordingly -## Run - -Jumpstart is containerized through a docker file. - +## Run 1. Build the docker file ``` docker build -t Jumpstart . @@ -37,11 +31,7 @@ Jumpstart is containerized through a docker file. docker run -p 8080:80 Jumpstart ``` -## Docker Compose - -Jumpstart also has support for Docker Compose, a extended version of docker that simplifies the steps. - -(This is a really cool thing! If you use docker often, check it out!) +## Alternatively, you can run the docker compose file as well ``` docker compose up ``` diff --git a/docker-compose.yml b/docker-compose.yml index 40dfeaf..00eef18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,5 +19,4 @@ services: - WIKI_API=${WIKI_API} - WIKIBOT_USER=${WIKIBOT_USER} - WIKIBOT_PASSWORD=${WIKIBOT_PASSWORD} - - WIKI_CATEGORY=${WIKI_CATEGORY} - \ No newline at end of file + - WIKI_CATEGORY=${WIKI_CATEGORY} \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index e32af29..c163f3e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,7 +18,7 @@ The application has multiple features: 3. A spot for announcements, which E-Board members can make through our Slack. This is done using a custom Slack app that utilizes Slack’s APIs. - 4. A section that displays a CSH wiki page for 20 seconds + 4. A section that displays a random title from r/showerthoughts every 30 seconds. 5. An informational that displays real-time status information from CSH’s server room. diff --git a/mkdocs.yml b/mkdocs.yml index acafb5b..6163132 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,7 +38,7 @@ use_directory_urls: false nav: - Home: index.md - Getting Started: getting-started/getting-started.md - - Backend: + - Core: - Wikithoughts: core/Wiki-thoughts.md - Calendar: core/CSH Calendar.md - Slack: core/Slack.md @@ -47,6 +47,8 @@ nav: - Announcements: endpoints/announcements.md - Slack Bot: endpoints/slack_bot.md - Wiki Thoughts: endpoints/wikithoughts.md + - Scripts: + - Setup: getting-started/setup.md plugins: - search diff --git a/src/api/endpoints.py b/src/api/endpoints.py index 8ece6b2..ef41487 100644 --- a/src/api/endpoints.py +++ b/src/api/endpoints.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Request, Form from fastapi.responses import JSONResponse -from core import slack, wikithoughts, cshcalendar +from core import slack, cshcalendar, wikithoughts from config import WATCHED_CHANNELS logger: Logger = getLogger(__name__) @@ -74,7 +74,7 @@ async def slack_events(request: Request) -> JSONResponse: if event.get("subtype", None) is not None: return JSONResponse({"status": "ignored"}) - if event not in WATCHED_CHANNELS: + if not event.get("channel", "") in WATCHED_CHANNELS: return JSONResponse({"status": "ignored"}) await slack.request_upload_via_dm(event.get("user", ""), cleaned_text) @@ -107,11 +107,9 @@ async def message_actions(payload: str = Form(...)) -> JSONResponse: if slack.convert_user_response_to_bool(form_json): logger.info("User approved the announcement!") logger.info(f"{form_json}\n\n") - message_object: dict[str, dict] = json.loads( - form_json.get("actions", [{}])[0].get("value", '{text:""}') - ).get("text", None) - logger.info(f"Display Object {message_object}") - slack.add_announcement(message_object) + messageObject = json.loads(form_json.get("actions",[{}])[0].get("value",'{text:""}')).get("text",None) + logger.info(f"Display Object {messageObject}") + slack.add_announcement(messageObject) if response_url: await httpx.post( diff --git a/src/config.py b/src/config.py index b641377..7e6a685 100644 --- a/src/config.py +++ b/src/config.py @@ -24,7 +24,7 @@ WIKI_CATEGORY: str = os.getenv("WIKI_CATEGORY", "JobAdvice") if SLACK_API_TOKEN in (None, ""): - raise ValueError("Missing SLACK_API_TOKEN") + raise Exception("Missing SLACK_API_TOKEN") with open(os.path.join(BASE_DIR, "static", "slack", "dm_request_template.json")) as f: SLACK_DM_TEMPLATE = json.load(f) diff --git a/src/core/cshcalendar.py b/src/core/cshcalendar/__init__.py similarity index 76% rename from src/core/cshcalendar.py rename to src/core/cshcalendar/__init__.py index 3434ff8..7005771 100644 --- a/src/core/cshcalendar.py +++ b/src/core/cshcalendar/__init__.py @@ -2,11 +2,11 @@ from datetime import datetime, date, timedelta, time from zoneinfo import ZoneInfo +# from icalendar import Calendar from icalendar.cal import Event, Calendar import httpx import recurring_ical_events import arrow -import re from config import ( CALENDAR_CACHE_REFRESH, @@ -34,29 +34,6 @@ logger.info("Starting up the calendar service!") cshcal_client = httpx.AsyncClient() -# Conversion from seconds -MINUTE: int = 60 -HOUR: int = MINUTE * 60 -DAY: int = HOUR * 24 -WEEK: int = DAY * 7 - -""" -This is used for each "check" from the time humanizer. %TIME% will be replaced with a rounded -WARNING: PERCENTAGE SIGNS WILL TRIGGER A REGEX OPERATION -WARNING: FOLLOW INSERTION ORDER -""" -HUMANIZER_CHECKS: dict[int, str] = { - MINUTE: "In 1 Minute", - (HOUR - MINUTE): f"In %{MINUTE}% Minutes", - (HOUR * 1.5): "In 1 Hour", - (DAY - HOUR): f"In %{HOUR}% Hours", - (DAY * 1.33): "In 1 Day", - (WEEK - DAY): f"In %{DAY}% Days", -} - -BORDER_STRING: str = "
" -TIME_PATTERN = re.compile(r"%([^%]+)%") - # Automatically format all info into the class class CalendarInfo: @@ -79,62 +56,6 @@ def __hash__(self): return hash((self.name, self.date)) -def ceil_division(num: int, den: int) -> int: - """ - Returns a ceiling division of the two numbers - Args: - num (int): the numerator - den (int): the denominator - - Returns: - int: the result of the operation - """ - return (num + den - 1) // den - - -def time_humanizer(current_time: datetime, event_time: datetime) -> str: - """ - Custom humanizer for text to be displayed - - Args: - current_time (datetime): The current time to be judged off of - event_time (datetime): The events time to be factored - Returns: - str: The humanized time as a string - """ - - def repl(match: re.Match[str]) -> str: - """ - Replaces the matched group text - - Args: - match (re.Match[str]): The matches group - - Returns: - str: The newly formatted string - """ - num = int(match.group(1)) - return str(round(time_before_event / num)) - - time_before_event: int = (event_time - current_time).total_seconds() - - if time_before_event > WEEK: - return "Over a Week Away" - - unformatted_string: str = ( - "------" # Make this the default, incase an operation fails - ) - - for key in HUMANIZER_CHECKS.keys(): - if time_before_event < key: - unformatted_string = HUMANIZER_CHECKS.get( - key, "Unable to find appropriate humanizer text" - ) - break - - return TIME_PATTERN.sub(repl, unformatted_string) - - def calendar_to_html(seg_header: str, seg_content: str) -> str: """ Formats a header and content into the HTML for the calendar front end @@ -172,11 +93,11 @@ def format_events(events: tuple[CalendarInfo]) -> dict[str, str]: final_events: str = "
" if not events: - final_events += BORDER_STRING + final_events += "
" final_events += calendar_to_html(":(", "No Events on the Calendar") - final_events += BORDER_STRING + final_events += "
" return {"data": final_events} @@ -190,11 +111,9 @@ def format_events(events: tuple[CalendarInfo]) -> dict[str, str]: ) final_events += calendar_to_html(formatted, event.name) else: - final_events += calendar_to_html( - time_humanizer(current_date, event.date), event.name - ) + final_events += calendar_to_html(event.date.humanize().title(), event.name) - final_events += BORDER_STRING + final_events += "
" return {"data": final_events} @@ -314,7 +233,7 @@ async def get_future_events() -> list[CalendarInfo]: if cal_correct_length: logger.info("Calendar cache is full length, rebuilding async!") - asyncio.create_task( + running_task = asyncio.create_task( rebuild_calendar() ) # Calendar is correct length, we can just run this in the background # Make it a variable for GC purposes? idk sonarqube told me to do it @@ -337,5 +256,6 @@ async def close_client() -> None: try: await cshcal_client.aclose() logger.info("Succesfully closed the cshcal client") - except RuntimeError: + except RuntimeError as e: logger.warning("EVENT LOOP HAS ALREADY CLOSED, FAILED TO CLOSE csh_cal") + return diff --git a/src/core/slack.py b/src/core/slack.py index b7fda91..754d6de 100644 --- a/src/core/slack.py +++ b/src/core/slack.py @@ -73,12 +73,10 @@ async def request_upload_via_dm(user_id: str, announcement_text: str) -> None: message: dict = copy.deepcopy(SLACK_DM_TEMPLATE) message[0]["text"]["text"] += announcement_text - message[1]["elements"][0]["value"] = json.dumps( - { - "text": announcement_text, - "user": user_id, - } - ) + message[1]["elements"][0]["value"] = json.dumps({ + "text": announcement_text, + "user": user_id, + }) await client.chat_postMessage( channel=user_id, text=SLACK_JUMPSTART_MESSAGE, blocks=message diff --git a/src/core/wikithoughts.py b/src/core/wikithoughts.py index cfd10b8..9894ace 100644 --- a/src/core/wikithoughts.py +++ b/src/core/wikithoughts.py @@ -2,7 +2,6 @@ import asyncio from datetime import datetime, timedelta from itertools import islice -from typing import Pattern from config import WIKIBOT_PASSWORD, WIKIBOT_USER, WIKI_CATEGORY, WIKI_API import logging import random @@ -33,18 +32,6 @@ page_last_updated: datetime | None = None -# Precompile all the Regex operations -RE_LINK: Pattern[str] = re.compile(r'\[https?://[^\s"]+\s+"?([^\]"]+)"?\]') -RE_FILE: Pattern[str] = re.compile(r"\[\[File:[^\]]*\]\]", re.IGNORECASE) -RE_IMAGE: Pattern[str] = re.compile(r"\[\[Image:[^\]]*\]\]", re.IGNORECASE) -RE_PAGE_TEXT: Pattern[str] = re.compile(r"\[\[[^\|\]]*\|([^\]]+)\]\]") -RE_PAGE: Pattern[str] = re.compile(r"\[\[([^\]]+)\]\]") -RE_CSH: Pattern[str] = re.compile(r"\^\^([^\]]+)\^\^") -RE_TEMPLATE: Pattern[str] = re.compile(r"\{\{.*?\}\}", re.DOTALL) -RE_HTML: Pattern[str] = re.compile(r"<[^>]+>") -RE_BOLD_ITALIC: Pattern[str] = re.compile(r"''+") - - def clean_wikitext(text: str) -> str: """ Function for cleaning markdown text using regex commands @@ -56,30 +43,31 @@ def clean_wikitext(text: str) -> str: str: The cleaned up text string """ - reg_operations: tuple[Pattern[str]] = ( - RE_LINK, - RE_FILE, - RE_IMAGE, - RE_PAGE_TEXT, - RE_PAGE, - RE_CSH, - RE_TEMPLATE, - RE_HTML, - RE_BOLD_ITALIC, - ) + # [https... Text] -> Text + text = re.sub(r"\[https?:\/\/\S+\s+\"?([^\]\"]+)\"?\]", r"\1", text) + + # [[File: Page|Text]] → Text + text = re.sub(r"\[\[File:[^\]]*\]\]", "", text, flags=re.IGNORECASE) + + # [[Image: Page|Text]] -> Text + text = re.sub(r"\[\[Image:[^\]]*\]\]", "", text, flags=re.IGNORECASE) + # [[Page|Text]] -> Text + text = re.sub(r"\[\[[^\|\]]*\|([^\]]+)\]\]", r"\1", text) - for operation in reg_operations: - swap_text: str = "" - if operation in ( - RE_LINK, - RE_PAGE, - RE_PAGE_TEXT, - ): # Keep text inbetween the anchors - swap_text = r"\1" - elif operation == (RE_CSH): # Add user infront of the CSH user - swap_text = r"User \1" + # [[Page]] -> Page + text = re.sub(r"\[\[([^\]]+)\]\]", r"\1", text) - text = operation.sub(swap_text, text) + # ^^CSH Account^^ -> User CSH Account + text = re.sub(r"\^\^([^\]]+)\^\^", r"User \1", text) + + # Remove templates {{...}} + text = re.sub(r"\{\{.*?\}\}", "", text, flags=re.DOTALL) + + # Remove HTML tags + text = re.sub(r"<.*?>", "", text) + + # Remove bold/italic markup + text = re.sub(r"''+", "", text) return text.strip() @@ -133,100 +121,35 @@ async def auth_bot() -> None: logger.warning("Bot was unable to authenticate!") -def headers_formatting( - new_etag: str | None = None, new_last_modified: str | None = None -) -> dict[str, str]: - """ - Formats and returns a header file for a wikithought request - - Args: - new_etag(str | None): The optional new etag to be globalized - new_last_modified(str | None): The optional new last modified to be globalized - - Returns: - dict[str,str]: The new headers to be applied - """ - global etag, last_modifed - - headers: dict[str, str] = {} - - if new_etag: - etag = new_etag - - if new_last_modified: - last_modifed = new_last_modified - - if etag: - headers["If-None-Match"] = etag - if last_modifed: - headers["If-Modified-Since"] = last_modifed - - return headers - - -def needs_category_refresh(update_time: datetime) -> bool: +async def refresh_category_pages() -> list[str]: """ - Verifys if the wikithoughts needs to be updated, checking bot status, cache and time + Refreshes all pages of the category Args: - update_time (datetime): The datetime to be compared against the cache + category (str): The name of the category to search through Returns: - boolean: if the cache needs to be refreshed + list[str]: All the page titles found in this category, None if the bot is not authorized """ + global \ + page_title_cache, \ + last_updated_time, \ + etag, \ + last_modifed, \ + queued_pages, \ + shown_pages if not bot_authenticated: logger.warning("Bot is not authenticated, cancelling fetch of category pages") - return False + return - return not ( + time_now: datetime = datetime.now() + if ( len(page_title_cache) > 0 and last_updated_time - and update_time < last_updated_time + timedelta(minutes=10) - ) - - -def process_category_page(response: httpx.Response) -> tuple[list[str], bool | str]: - """ - Processes a wikithoughts response into a list of title pages - - Args: - respone (httpx.Response): The response from the wiki to be processed - - Returns: - tuple[list[str], bool | str]: The list of titles from the request, along with a possible continutation if needed - """ - r_json: dict[str, str] = response.json() - titles: list[str] = [] - if "query" in r_json: - for page in r_json["query"]["categorymembers"]: - titles.append(page["title"]) - - # Loop to keep everything going - if "continue" in r_json: - return (titles, r_json["continue"]["cmcontinue"]) - - return (titles, False) - else: - logger.warning(f"Failure in obtaining info, JSON:\n{r_json}") - return (titles, False) - - -async def refresh_category_pages() -> list[str]: - """ - Refreshes all pages of the category - - Args: - category (str): The name of the category to search through - - Returns: - list[str]: All the page titles found in this category, None if the bot is not authorized - """ - global page_title_cache, last_updated_time, queued_pages, shown_pages - time_now: datetime = datetime.now() - - if not needs_category_refresh(time_now): - return page_title_cache + and time_now < last_updated_time + timedelta(minutes=10) + ): + return titles: list[str] = [] params: dict[str, str] = { @@ -237,9 +160,16 @@ async def refresh_category_pages() -> list[str]: "format": "json", } - headers: dict[str, str] = headers_formatting() + headers: dict[str, str] = {} # This needs to loop due to mediawiki limitations while True: + if not "cmcontinue" in params: + if etag: + headers["If-None-Match"] = etag + if last_modifed: + headers["If-Modified-Since"] = last_modifed + else: + headers = {} response: httpx.Response = await client.get( WIKI_API, params=params, headers=headers ) @@ -249,14 +179,22 @@ async def refresh_category_pages() -> list[str]: return page_title_cache elif response.status_code == 200: - headers_formatting(etag, last_modifed) - added, repeat_req = process_category_page(response=response) - titles += added - - if repeat_req not in (None, False, ""): - params["cmcontinue"] = repeat_req - continue - break + etag = response.headers.get("ETag") + last_modifed = response.headers.get("Last-Modified") + + r_json: dict[str, str] = response.json() + if "query" in r_json: + for page in r_json["query"]["categorymembers"]: + titles.append(page["title"]) + + # Loop to keep everything going + if "continue" in r_json: + params["cmcontinue"] = r_json["continue"]["cmcontinue"] + else: + break + else: + logger.warning(f"Failure in obtaining info, JSON:\n{r_json}") + break else: logger.warning("Failed to update the CSH wiki page!") return page_title_cache @@ -264,7 +202,6 @@ async def refresh_category_pages() -> list[str]: last_updated_time = time_now page_title_cache = titles queued_pages = titles.copy() - random.shuffle(queued_pages) shown_pages = [] @@ -275,11 +212,12 @@ async def refresh_category_pages() -> list[str]: async def refresh_page_dictionary() -> None: """ Fetches the pages based off the cache of page titles, and updates the page cache accordingly + """ global page_dict_cache, page_title_cache if not page_title_cache: - return + return {} results: dict[str, str] = {} tasks: list = [] @@ -325,7 +263,7 @@ def reset_queues() -> None: Swaps Queued and Shown pages queued """ global queued_pages, shown_pages - logger.info("RESETING QUEUES FOR WIKITHOUGHTS") + logger.warning("RESETING QUEUES FOR WIKITHOUGHTS") if len(queued_pages) > 0: return @@ -354,13 +292,12 @@ async def get_next_display() -> dict[str, str]: queue_empty: bool = len(queued_pages) == 0 if queue_empty and len(shown_pages) == 0: - logger.warning("Error, queue and shown pages are both empty!") + logger.warning("ERROR?!?") current_page = { "page": "ERROR GETTING PAGE", "content": "ERROR FETCHING CONTENT", } - return current_page - + return elif queue_empty: reset_queues() queue_empty = False diff --git a/src/main.py b/src/main.py index ed3b131..861d12f 100644 --- a/src/main.py +++ b/src/main.py @@ -14,12 +14,13 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from fastapi.responses import RedirectResponse, HTMLResponse + from contextlib import asynccontextmanager from config import BASE_DIR from api import endpoints -from core import wikithoughts, cshcalendar +from core import cshcalendar, wikithoughts logger: Logger = getLogger(__name__)