diff --git a/.github/workflows/build_exe.yml b/.github/workflows/build_exe.yml index db6832c7..b9a3b609 100644 --- a/.github/workflows/build_exe.yml +++ b/.github/workflows/build_exe.yml @@ -2,6 +2,18 @@ name: Build Executables on: workflow_call: + secrets: + APPLE_DEV_EMAIL: + APP_SPEC_PASS: + DEV_APP_CERT: + DEV_APP_CERT_PASS: + DEV_INST_CERT: + DEV_INST_CERT_PASS: + APPLE_KEY_PASS: + APPLE_APP_CERT_ID: + APPLE_INST_CERT_ID: + TEAM_ID: + inputs: is_release: default: false @@ -62,7 +74,7 @@ jobs: echo "PYINST_BIN=\"$(which pyinstaller)\"" >> "$GITHUB_ENV" - name: Create macOS keychain id: keychain - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release env: APPLE_DEV_EMAIL: ${{secrets.APPLE_DEV_EMAIL}} APP_SPEC_PASS: ${{secrets.APP_SPEC_PASS}} @@ -113,7 +125,7 @@ jobs: run: ${{ matrix.sha_command }} - name: Create and sign mac package - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release env: APPLE_INST_CERT_ID: ${{secrets.APPLE_INST_CERT_ID}} APPLE_KEYCHAIN_PASS: ${{secrets.APPLE_KEY_PASS}} @@ -127,19 +139,19 @@ jobs: pkgbuild --identifier "org.mangotango.cli" --timestamp --root /tmp/mangotango --install-location /Applications "./dist/mangotango_${{matrix.artifact_name}}_signed.pkg" --sign "$APPLE_INST_CERT_ID" - name: Notarize Mac package - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release env: APPLE_DEV_EMAIL: ${{secrets.APPLE_DEV_EMAIL}} APPLE_TEAM_ID: ${{secrets.TEAM_ID}} APP_SPEC_PASS: ${{secrets.APP_SPEC_PASS}} - run: xcrun notarytool submit dist/mangotango_${{matrix.artifact_name}}_signed.pkg --apple-id $APPLE_DEV_EMAIL --team-id $APPLE_TEAM_ID --password $APP_SPEC_PASS --wait > notarization_output.txt + run: xcrun notarytool submit dist/mangotango_${{matrix.artifact_name}}_signed.pkg --apple-id $APPLE_DEV_EMAIL --team-id $APPLE_TEAM_ID --password $APP_SPEC_PASS --wait - name: Staple the notarization ticket - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release run: xcrun stapler staple dist/mangotango_${{matrix.artifact_name}}_signed.pkg - name: Clean up macOS Artifacts - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release run: | rm -rf /tmp/mangotango rm -rf dist/mangotango_${{matrix.artifact_name}} @@ -147,7 +159,7 @@ jobs: mv dist/mangotango_${{matrix.artifact_name}}_signed.pkg dist/mangotango_${{matrix.artifact_name}}.pkg - name: Compute the SHA1 hashsum for macOS .pkg - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.is_release run: ${{ matrix.sha_command_pkg }} - name: Inspect the dist/ directory before uploading artifacts @@ -160,4 +172,3 @@ jobs: name: ${{ matrix.artifact_name }} path: | dist/* - diff --git a/analyzer_interface/context.py b/analyzer_interface/context.py index 4f3a8998..4932f05e 100644 --- a/analyzer_interface/context.py +++ b/analyzer_interface/context.py @@ -1,9 +1,12 @@ from abc import ABC, abstractmethod -from typing import TypeVar +from typing import Any, Callable, Optional, TypeVar, Union import polars as pl from dash import Dash +from polars import DataFrame from pydantic import BaseModel +from shiny import Inputs, Outputs, Session +from shiny.ui._navs import NavPanel from .interface import SecondaryAnalyzerInterface from .params import ParamValue @@ -155,3 +158,25 @@ def parquet_path(self) -> str: file to it. """ pass + + +ServerCallback = Union[ + Callable[[Inputs], None], Callable[[Inputs, Outputs, Session], None] +] + + +class ShinyContext(BaseModel): + panel: NavPanel = None + server_handler: Optional[ServerCallback] = None + + class Config: + arbitrary_types_allowed = True + + +class FactoryOutputContext(BaseModel): + shiny: Optional[ShinyContext] = None + api: Optional[dict[str, Any]] = None + data_frames: Optional[dict[str, DataFrame]] = None + + class Config: + arbitrary_types_allowed = True diff --git a/analyzer_interface/declaration.py b/analyzer_interface/declaration.py index d3e725f8..4209b32a 100644 --- a/analyzer_interface/declaration.py +++ b/analyzer_interface/declaration.py @@ -1,6 +1,7 @@ -from typing import Callable +from typing import Callable, Union from .context import ( + FactoryOutputContext, PrimaryAnalyzerContext, SecondaryAnalyzerContext, WebPresenterContext, @@ -69,11 +70,17 @@ def __init__(self, interface: SecondaryAnalyzerInterface, main: Callable): class WebPresenterDeclaration(WebPresenterInterface): - factory: Callable[["WebPresenterContext"], None] - + factory: Callable[["WebPresenterContext"], Union[FactoryOutputContext, None]] + shiny: bool server_name: str - def __init__(self, interface: WebPresenterInterface, factory: Callable, name: str): + def __init__( + self, + interface: WebPresenterInterface, + factory: Callable, + name: str, + shiny: bool, + ): """Creates a web presenter declaration Args: @@ -99,4 +106,6 @@ def __init__(self, interface: WebPresenterInterface, factory: Callable, name: st https://docs.python.org/3/tutorial/modules.html """ - super().__init__(**interface.model_dump(), factory=factory, server_name=name) + super().__init__( + **interface.model_dump(), factory=factory, server_name=name, shiny=shiny + ) diff --git a/analyzers/example/example_web/__init__.py b/analyzers/example/example_web/__init__.py index d6033220..be1825f0 100644 --- a/analyzers/example/example_web/__init__.py +++ b/analyzers/example/example_web/__init__.py @@ -9,4 +9,5 @@ # You must pass __name__ here. It's to make Dash happy. # See: http://dash.plotly.com/urls name=__name__, + shiny=False, ) diff --git a/analyzers/hashtags_web/__init__.py b/analyzers/hashtags_web/__init__.py index f1f5ea2d..ac337e2f 100644 --- a/analyzers/hashtags_web/__init__.py +++ b/analyzers/hashtags_web/__init__.py @@ -4,5 +4,5 @@ from .interface import interface hashtags_web = WebPresenterDeclaration( - interface=interface, factory=factory, name=__name__ + interface=interface, factory=factory, name=__name__, shiny=True ) diff --git a/analyzers/hashtags_web/app.py b/analyzers/hashtags_web/app.py index 9d43c302..d45fd5b4 100644 --- a/analyzers/hashtags_web/app.py +++ b/analyzers/hashtags_web/app.py @@ -2,23 +2,24 @@ import numpy as np import polars as pl -from shiny import App, reactive, render, ui +from shiny import reactive, render, ui from shinywidgets import output_widget, render_widget from ..hashtags.interface import COL_AUTHOR_ID, COL_POST, COL_TIME from .analysis import secondary_analyzer from .plots import plot_bar_plotly, plot_gini_plotly, plot_users_plotly -MANGO_ORANGE2 = "#f3921e" -LOGO_URL = "https://raw.githubusercontent.com/CIB-Mango-Tree/CIB-Mango-Tree-Website/main/assets/images/mango-text.PNG" - # https://icons.getbootstrap.com/icons/question-circle-fill/ question_circle_fill = ui.HTML( '' ) +df_global = None +context_global = None +df_raw = None + -def _set_df_global_state(df_input, df_output): +def set_df_global_state(df_input, df_output): global df_global, df_raw df_global = df_output df_raw = df_input # Will be loaded from context when needed @@ -36,9 +37,6 @@ def get_raw_data_subset(time_start, time_end, user_id, hashtag): # Global variables for CLI integration -df_global = None -context_global = None -df_raw = None def select_users(secondary_output, selected_hashtag): @@ -154,46 +152,6 @@ def select_users(secondary_output, selected_hashtag): ui.output_data_frame("tweets"), ) -analysis_panel_elements = [ - page_dependencies, - analysis_panel, - ui.layout_columns( - hashtag_plot_panel, - users_plot_panel, - ), - tweet_explorer, -] - - -ABOUT_TEXT = ui.markdown( - f""" - -logo - -CIB Mango Tree, a collaborative and open-source project to develop software that tests for coordinated inauthentic behavior (CIB) in datasets of online activity. - -[mangotree.org](https://mangotree.org) - -A project of [Civic Tech DC](https://www.civictechdc.org/), our mission is to share methods to uncover how disruptive actors seek to hack our legitimate online discourse regarding health, politics, and society. The CIB Mango Tree presents the most simple tests for CIB first – the low-hanging fruit. These tests are easy to run and interpret. They will reveal signs of unsophisticated CIB. As you move up the Mango Tree, tests become harder and will scavenge for higher-hanging fruit. - -""" -) -app_layout = ui.page_navbar( - ui.nav_panel( - "Dashboard", - analysis_panel_elements, - ), - ui.nav_panel( - "About", - ui.card( - ui.card_header("About the Mango Tree project"), - ABOUT_TEXT, - ui.card_footer("PolyForm Noncommercial License 1.0.0"), - ), - ), - title="Hashtag analysis dashboard", -) - def server(input, output, session): @reactive.calc diff --git a/analyzers/hashtags_web/factory.py b/analyzers/hashtags_web/factory.py index 8b73788b..aff5b625 100644 --- a/analyzers/hashtags_web/factory.py +++ b/analyzers/hashtags_web/factory.py @@ -1,26 +1,28 @@ import polars as pl -from pydantic import BaseModel +from dash import html +from shiny.ui import layout_columns, nav_panel -from analyzer_interface.context import WebPresenterContext +from analyzer_interface.context import ( + FactoryOutputContext, + ShinyContext, + WebPresenterContext, +) +from app.shiny import page_dependencies from ..hashtags.interface import COL_AUTHOR_ID, COL_POST, COL_TIME, OUTPUT_GINI -from .app import _set_df_global_state, app_layout, server - - -# config used by AnalysisWebServerContext._start_shiny_server() -class ShinyServerConfig(BaseModel): - server: object - port: int = 8051 - - -class ShinyAppConfig(BaseModel): - app_layout: object - server_config: ShinyServerConfig +from .app import ( + analysis_panel, + hashtag_plot_panel, + server, + set_df_global_state, + tweet_explorer, + users_plot_panel, +) def factory( web_context: WebPresenterContext, -): +) -> FactoryOutputContext: # Load the primary output associated with this project df_hashtags = pl.read_parquet(web_context.base.table(OUTPUT_GINI).parquet_path) @@ -41,26 +43,33 @@ def factory( pl.col(COL_TIME) ) - # Create app layout and server with processed data - _set_df_global_state(df_input=df_raw, df_output=df_hashtags) - - # For now, mount the Shiny app via iframe in Dash - # This preserves the existing CLI integration - from dash import html - - app = web_context.dash_app + set_df_global_state( + df_input=df_raw, + df_output=df_hashtags, + ) - app.layout = html.Div( + web_context.dash_app.layout = html.Div( [ html.H1("Hashtag Analysis Dashboard"), html.Iframe( - src="http://localhost:8051", # Shiny app will run on port 8051 + src="http://localhost:8050/shiny", # Shiny app will run on port 8051 style={"width": "100%", "height": "800px", "border": "none"}, ), ] ) - # Return Shiny app configuration for the webserver context to handle - return ShinyAppConfig( - app_layout=app_layout, server_config=ShinyServerConfig(server=server, port=8051) + return FactoryOutputContext( + shiny=ShinyContext( + server_handler=server, + panel=nav_panel( + "Dashboard", + page_dependencies, + analysis_panel, + layout_columns( + hashtag_plot_panel, + users_plot_panel, + ), + tweet_explorer, + ), + ) ) diff --git a/analyzers/ngram_web/__init__.py b/analyzers/ngram_web/__init__.py index efeab491..8641a856 100644 --- a/analyzers/ngram_web/__init__.py +++ b/analyzers/ngram_web/__init__.py @@ -4,5 +4,5 @@ from .interface import interface ngrams_web = WebPresenterDeclaration( - interface=interface, factory=factory, name=__name__ + interface=interface, factory=factory, name=__name__, shiny=False ) diff --git a/analyzers/temporal_barplot/__init__.py b/analyzers/temporal_barplot/__init__.py index 0c009c48..e23dd440 100644 --- a/analyzers/temporal_barplot/__init__.py +++ b/analyzers/temporal_barplot/__init__.py @@ -4,5 +4,5 @@ from .interface import interface temporal_barplot = WebPresenterDeclaration( - interface=interface, factory=factory, name=__name__ + interface=interface, factory=factory, name=__name__, shiny=False ) diff --git a/app/analysis_webserver_context.py b/app/analysis_webserver_context.py index 191f83a1..19f7948f 100644 --- a/app/analysis_webserver_context.py +++ b/app/analysis_webserver_context.py @@ -1,39 +1,53 @@ -import logging -import os.path -import threading +from os import path from pathlib import Path from tempfile import TemporaryDirectory +from a2wsgi import WSGIMiddleware from dash import Dash from flask import Flask, render_template from pydantic import BaseModel -from shiny import App -from waitress import serve -from werkzeug.middleware.dispatcher import DispatcherMiddleware +from shiny import App, ui +from starlette.applications import Starlette +from starlette.responses import RedirectResponse +from starlette.routing import Mount, Route +from uvicorn import Config, Server from context import WebPresenterContext from .analysis_context import AnalysisContext from .app_context import AppContext +from .shiny import LayoutManager, ServerHandleManager class AnalysisWebServerContext(BaseModel): app_context: AppContext analysis_context: AnalysisContext - _shiny_threads: list = [] def start(self): containing_dir = str(Path(__file__).resolve().parent) - static_folder = os.path.join(containing_dir, "web_static") - template_folder = os.path.join(containing_dir, "web_templates") - + static_folder = path.join(containing_dir, "web_static") + template_folder = path.join(containing_dir, "web_templates") web_presenters = self.analysis_context.web_presenters + project_name = self.analysis_context.project_context.display_name + analyzer_name = self.analysis_context.display_name + server_handler_manager = ServerHandleManager() + layout_manager = LayoutManager() web_server = Flask( __name__, template_folder=template_folder, static_folder=static_folder, static_url_path="/static", ) + + @web_server.route("/") + def index(): + return render_template( + "index.html", + panels=[(presenter.id, presenter.name) for presenter in web_presenters], + project_name=project_name, + analyzer_name=analyzer_name, + ) + web_server.logger.disabled = True temp_dirs: list[TemporaryDirectory] = [] @@ -41,69 +55,52 @@ def start(self): dash_app = Dash( presenter.server_name, server=web_server, - url_base_pathname=f"/{presenter.id}/", - external_stylesheets=["/static/dashboard_base.css"], + requests_pathname_prefix=f"/dash/{presenter.id}/", + routes_pathname_prefix=f"/{presenter.id}/", + external_stylesheets=["/dash/static/dashboard_base.css"], ) temp_dir = TemporaryDirectory() - presenter_context = WebPresenterContext( analysis=self.analysis_context.model, web_presenter=presenter, store=self.app_context.storage, temp_dir=temp_dir.name, dash_app=dash_app, - shiny_app=None, ) temp_dirs.append(temp_dir) result = presenter.factory(presenter_context) - # Handle Shiny app if returned by factory - if presenter.id == "hashtags_dashboard": - # initiate Shiny app instance - presenter_context.shiny_app = App( - ui=result.app_layout, server=result.server_config.server - ) + if result is None or result.shiny is None: + continue - self._start_shiny_server( - shiny_app=presenter_context.shiny_app, - port=result.server_config.port, - ) + server_handler_manager.add(result.shiny.server_handler) + layout_manager.add(result.shiny.panel) - project_name = self.analysis_context.project_context.display_name - analyzer_name = self.analysis_context.display_name - - @web_server.route("/") - def index(): - return render_template( - "index.html", - panels=[(presenter.id, presenter.name) for presenter in web_presenters], - project_name=project_name, - analyzer_name=analyzer_name, - ) + async def relay(_): + return RedirectResponse("/shiny" if web_presenters[0].shiny else "/dash") - app_dispatcher = DispatcherMiddleware(web_server) - server_log = logging.getLogger("waitress") - original_log_level = server_log.level - original_disabled = server_log.disabled - server_log.setLevel(logging.ERROR) - server_log.disabled = True + shiny_app = App( + ui=layout_manager.build_layout(), + server=server_handler_manager.call_handlers, + debug=False, + ) + app = Starlette( + debug=False, + routes=[ + Route("/", relay), + Mount("/dash", app=WSGIMiddleware(web_server), name="dash_app"), + Mount("/shiny", app=shiny_app, name="shiny_app"), + ], + ) try: - server_log.setLevel(original_log_level) - server_log.disabled = original_disabled - - serve(app_dispatcher, host="127.0.0.1", port=8050) - finally: - for temp_dir in temp_dirs: - temp_dir.cleanup() + config = Config(app, host="0.0.0.0", port=8050, log_level="error") + uvi_server = Server(config) - def _start_shiny_server(self, shiny_app, port=8051): - """Start a Shiny server in a separate thread""" - from shiny import run_app + uvi_server.run() - def run_shiny(): - run_app(shiny_app, host="127.0.0.1", port=port, launch_browser=True) + except Exception as err: + print(err) - shiny_thread = threading.Thread(target=run_shiny, daemon=True) - shiny_thread.start() - self._shiny_threads.append(shiny_thread) + for temp_dir in temp_dirs: + temp_dir.cleanup() diff --git a/app/shiny.py b/app/shiny.py new file mode 100644 index 00000000..a159287a --- /dev/null +++ b/app/shiny.py @@ -0,0 +1,92 @@ +from inspect import signature +from typing import List, Optional + +from pydantic import BaseModel +from shiny.session import Inputs, Outputs, Session +from shiny.ui import ( + _navs, + card, + card_footer, + card_header, + markdown, + nav_panel, + page_navbar, + tags, +) + +from analyzer_interface.context import ServerCallback + +MANGO_ORANGE2 = "#f3921e" +LOGO_URL = "https://raw.githubusercontent.com/CIB-Mango-Tree/CIB-Mango-Tree-Website/main/assets/images/mango-text.PNG" +ABOUT_TEXT = markdown( + f""" + +logo + +CIB Mango Tree, a collaborative and open-source project to develop software that tests for coordinated inauthentic behavior (CIB) in datasets of online activity. + +[mangotree.org](https://mangotree.org) + +A project of [Civic Tech DC](https://www.civictechdc.org/), our mission is to share methods to uncover how disruptive actors seek to hack our legitimate online discourse regarding health, politics, and society. The CIB Mango Tree presents the most simple tests for CIB first – the low-hanging fruit. These tests are easy to run and interpret. They will reveal signs of unsophisticated CIB. As you move up the Mango Tree, tests become harder and will scavenge for higher-hanging fruit. + +""" +) +page_dependencies = tags.head( + tags.style(".card-header { color:white; background:#f3921e !important; }"), + tags.link(rel="stylesheet", href="https://fonts.googleapis.com/css?family=Roboto"), + tags.style("body { font-family: 'Roboto', sans-serif; }"), +) + + +class ServerHandleManager(BaseModel): + handlers: Optional[list[ServerCallback]] = [] + + def add(self, handler: ServerCallback): + self.handlers.append(handler) + + def extend(self, handlers: List[ServerCallback]): + self.handlers.extend(handlers) + + def remove(self, handler: ServerCallback): + self.handlers.remove(handler) + + def call_handlers(self, inputs: Inputs, outputs: Outputs, session: Session): + for handler in self.handlers: + handler_signature = signature(handler) + + if len(handler_signature.parameters) == 1: + handler(inputs) + continue + + handler(inputs, outputs, session) + + +class LayoutManager(BaseModel): + title: Optional[str] = "Mango Tango Dashboard" + elements: Optional[List[_navs.NavPanel]] = [] + + class Config: + arbitrary_types_allowed = True + + def add(self, element: _navs.NavPanel): + self.elements.append(element) + + def extend(self, elements: List[_navs.NavPanel]): + self.elements.extend(elements) + + def remove(self, element: _navs.NavPanel): + self.elements.remove(element) + + def build_layout(self): + return page_navbar( + *self.elements, + nav_panel( + "About", + card( + card_header("About the Mango Tree project"), + ABOUT_TEXT, + card_footer("PolyForm Noncommercial License 1.0.0"), + ), + ), + title=self.title, + ) diff --git a/app/web_templates/index.html b/app/web_templates/index.html index c4189c71..38d44d55 100644 --- a/app/web_templates/index.html +++ b/app/web_templates/index.html @@ -96,7 +96,7 @@

Analysis: {{ analyzer_name }}

function select_panel(id) { const frame = document.getElementById('dashboard_frame'); - frame.src = `/${id}/`; + frame.src = `/dash/${id}/`; } select_panel('{{ panels[0][0] }}'); diff --git a/context/__init__.py b/context/__init__.py index ee9f33fd..7ee502bd 100644 --- a/context/__init__.py +++ b/context/__init__.py @@ -178,7 +178,6 @@ class WebPresenterContext(BaseWebPresenterContext): web_presenter: WebPresenterInterface store: Storage dash_app: Dash - shiny_app: object = None class Config: arbitrary_types_allowed = True diff --git a/requirements.txt b/requirements.txt index 70b92513..815a6caa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,10 @@ plotly==5.24.1 pandas==2.2.3 # needed by plotly pyarrow==17.0.0 dash==2.18.1 -waitress==3.0.2 colorama==0.4.6 fastexcel==0.13.0 shiny==1.4.0 -shinywidgets==0.6.2 \ No newline at end of file +shinywidgets==0.6.2 +starlette==0.47.1 +uvicorn==0.34.3 +a2wsgi==1.10.10 \ No newline at end of file