diff --git a/.gitignore b/.gitignore index f8c29f0..65fa507 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *~ *.log -*.yaml lib64 pip-selfcheck.json @@ -9,6 +8,7 @@ throttle.ctrl user-password.py user-config.py pyvenv.cfg +.env __pycache__/ bin/ diff --git a/README.md b/README.md index e1a50be..34f9d45 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,16 @@ This bot adds initial information at the start of year for Wikimedia Sverige. It # Installation Install required libraries with pip: - -`$ pip install -r requirements.txt` +``` +pip install -r requirements.txt +``` # Configuration A configuration file, `config.yaml` by default, is needed to run. `config.yaml.sample` has comments documenting the various parameters and can be used as a template. Configure Pywikibot, following the instructions on https://www.mediawiki.org/wiki/Manual:Pywikibot/user-config.py. -To edit Phabricator, a Conduit API token is required. This can be generated in the user settings: `Settings` -> `Conduit API Tokens`. +To edit Phabricator, a Conduit API token is required. This can be generated in the user settings: `Settings` -> `Conduit API Tokens`. The API token is read from environment variable `PHAB_API_TOKEN` which can be specified in the file `.env`. # Input Data ## Files @@ -20,11 +21,15 @@ Two files are required to run the bot: one containing project information and on Information about programs, strategies and goals are fetched from a table on a wikipage, such as [this one from 2019](https://se.wikimedia.org/w/index.php?title=Verksamhetsplan_2019/Tabell_%C3%B6ver_program,_strategi_och_m%C3%A5l&oldid=75471). The code for this is specific for this particular table and would need to be adapted if the same information is represented in another way. Note that some information is stored in HTML comments. # Running -The bot is run with the command - - $ ./project_start.py project-file goal-file - -For flags, see command line help (`$ ./project_start.py --help`). +The bot is run with the command: +``` +./project_start.py project-file [goal-file] +``` + +For flags, see command line help: +``` +./project_start.py --help +``` ## Logging The most important log messages are written to standard out. If you want more detailed logging, see the log file *project-start.log*. diff --git a/config.yaml.wmse b/config.yaml similarity index 79% rename from config.yaml.wmse rename to config.yaml index 6d562c5..71a5132 100644 --- a/config.yaml.wmse +++ b/config.yaml @@ -6,10 +6,10 @@ project_columns: lead: Ansvarig e_mail: Mejlprefix project_number: Projektnummer - area: Program + program: Program swedish_name: Svenskt projektnamn english_name: Engelskt projektnamn - about_english: About the project + about_english: Mini-description of the project about_swedish: Om projektet project_start: Projektstart project_end: Projektslut @@ -46,8 +46,8 @@ wiki: budget: budget finansiär_2: funder_2 budget_2: budget_2 + year_parameter: år year_pages: - operational_plan: Verksamhetsplan_/Tabell_över_program,_strategi_och_mål current_projects_template: Mall:Aktuella projekt simple: : Mall:Årsida @@ -63,19 +63,6 @@ wiki: projects: title: Projekt template: Mall:Årsida för projekt - program_overview: - colours: - - "#339966" - - "#006699" - - "#990000" - - "#FF8C00" - title: Verksamhetsplan /Måluppfyllnad/Programöversikt - templates: - page: Mall:Programöversikt - program: Mall:Programöversikt/Program - strategy: Mall:Programöversikt/Strategi - goal: Mall:Programöversikt/Mål - project: Mall:Programöversikt/Projekt categories: general: Föreningen pages: @@ -86,7 +73,6 @@ wiki: Resor : Resor Årsmöte : Årsmöte Checklistor : Checklistor - Utbildning : Utbildning Globala mätetal : Globala mätetal volunteer_tasks: title: Frivilliguppdrag @@ -94,7 +80,6 @@ wiki: project_name_template: Mall:Projektnamn project_number_template: Mall:Projektid phab: - api_token: api-... api_url: https://phabricator.wikimedia.org/api parent_project_id: 2480 - request_delay: 10 + request_delay: 1 diff --git a/config.yaml.sample b/config.yaml.sample index 4b05908..3ddcf63 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -39,8 +39,6 @@ wiki: # put and the name of the template to use for the goals. template_parameter: Template:Goals year_pages: - # Page where the operational plan can be found. - operational_plan: # Template where current projects are listed current_projects_template: # Map of pages that will be created by substituting @@ -52,19 +50,6 @@ wiki: projects: title: template: - # Parameters for program overview page. - program_overview: - # Colours to use in the program headers. - colours: - - #123456 - title: - templates: - # Template that is substed to create the page. - page: - # Templates that are used by the page template. Maps - # parameter name in the page template to template - # name. - parameter: Template # Year categories. categories: # This category is added to each category page. @@ -85,8 +70,6 @@ wiki: project_number_template: # Parameters for Phabricator phab: - # API-token for Conduit. - api_token: api-... # URL to the API. api_url: https://.../api # Id of the project that created projects will be subprojects diff --git a/phab.py b/phab.py index a99190e..aacb6b3 100644 --- a/phab.py +++ b/phab.py @@ -1,7 +1,9 @@ import logging from time import sleep, time +import os import requests +from dotenv import load_dotenv class Phab: @@ -23,6 +25,8 @@ def __init__(self, config, dry_run): self._config = config self._dry_run = dry_run self._last_request_time = 0.0 + load_dotenv() + self._api_token = os.getenv("PHAB_API_TOKEN") def add_project(self, name, description): """Add project. @@ -131,7 +135,7 @@ def _make_request(self, endpoint, parameters_dict): parameters = self._to_phab_parameters(parameters_dict) # Add placeholder API token to not reveal the real one in logs. logged_parameters = parameters.copy() - logged_parameters["api.token"] = "api-..." + logged_parameters["api.token"] = "api-REDACTED" logging.debug( "POST to Phabricator API on {}/{}: {}".format( self._config["api_url"], @@ -139,7 +143,7 @@ def _make_request(self, endpoint, parameters_dict): logged_parameters ) ) - parameters["api.token"] = self._config["api_token"] + parameters["api.token"] = self._api_token self._last_request_time = time() response = requests.post( "{}/{}".format(self._config["api_url"], endpoint), diff --git a/project_start.py b/project_start.py index 93866b1..70385b4 100755 --- a/project_start.py +++ b/project_start.py @@ -291,7 +291,8 @@ def process_project(project_information, project_columns): wiki.add_project( project_information[project_columns["project_number"]], project_information[project_columns["swedish_name"]], - project_information[project_columns["english_name"]] + project_information[project_columns["english_name"]], + project_information[project_columns["program"]] ) diff --git a/requirements.txt b/requirements.txt index af05e07..5c0021e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pywikibot==8.1.2 pyyaml==6.0 mwparserfromhell==0.6.4 wikitables==0.5.5 +python-dotenv==1.0.0 diff --git a/wiki.py b/wiki.py index 63597a7..9a45597 100644 --- a/wiki.py +++ b/wiki.py @@ -51,7 +51,7 @@ def __init__( self._prompt_add_pages = prompt_add_pages self._site = Site() self._projects = {} - self._programs = None + self._programs = [] self._touched_pages = [] def add_project_page( @@ -103,8 +103,8 @@ def add_project_page( self._components is None or Components.CATEGORIES.value in self._components ): - area = self._project_columns["area"] - self.add_project_categories(name, area) + program = parameters[self._project_columns["program"]] + self._add_project_categories(name, program) def _add_project_main_page(self, parameters, phab_id, phab_name): """Add the main project page, i.e. "Projekt:NAME". @@ -274,26 +274,26 @@ def _create_goal_fulfillment_text(self, goals): ) return fulfillment_text - def add_project_categories(self, project, area): + def _add_project_categories(self, project, program): """Add categories to the project's category page. - Adds the project category to categories for year and area, if + Adds the project category to categories for year and program, if given. Parameters ---------- project : str The project name in Swedish. - area : str - The area category to add the project category to. If the - empty string, no area category is added. + program : str + The program category to add the project category to. If the + empty string, no program category is added. """ year_category = "Projekt {}".format(self._year) categories = [year_category] - if area: - categories.append(area) + if program: + categories.append(program) self._add_category_page(project, categories) def _add_category_page(self, title, categories): @@ -345,8 +345,6 @@ def add_year_pages(self): ) if self._prompt_add_page(year_pages["projects"]["title"]): self._add_projects_year_page() - if self._prompt_add_page(year_pages["program_overview"]["title"]): - self._add_program_overview_year_page() if self._prompt_add_page("categories"): self._add_year_categories() if self._prompt_add_page(year_pages["current_projects_template"]): @@ -402,23 +400,14 @@ def _add_projects_year_page(self): config = self._config["year_pages"]["projects"] title = self._make_year_title(config["title"]) content = "" - try: - programs = self._get_programs() - except PageMissingError as error: - logging.error(f"Error when processing '{title}'.") - raise error - for program in programs: - content += "== {} {} ==\n".format( - program["number"], - program["name"] - ) - for strategy in program["strategies"]: - content += "=== {} {} ===\n".format( - strategy["number"], - strategy["short_description"] - ) - for project in strategy["projects"]: - content += self._make_project_data_string(project) + + for program in self._programs: + content += "== {} ==\n".format(program) + for project_number, project in self._projects.items(): + if project["program"] == program: + content += self._make_project_data_string(project_number) + content += "\n" + self._add_page_from_template( None, title, @@ -479,8 +468,8 @@ def _make_project_data_string(self, project): comment = Template("Utkommenterat", True, [project]) return "{}{}\n".format(project_template, comment) - def add_project(self, number, swedish_name, english_name): - """Store project number and name, Swedish and English, in a map. + def add_project(self, number, swedish_name, english_name, program): + """Store project information in a map. Parameters ---------- @@ -490,167 +479,17 @@ def add_project(self, number, swedish_name, english_name): Swedish project name. english_name : str English project name. + program : str + Which program the project is under. """ self._projects[number] = { "sv": swedish_name, - "en": english_name + "en": english_name, + "program": program } - - def _get_programs(self): - """Get descriptions for program, strategies and names. - - Parses a table from the wiki or just return the result if it - has been parsed already. Assumes a wikipage with a table - formatted in a particular way, with cells spanning multiple - rows and HTML comments containing some of the information. An - instance of such a table can be found on: - https://se.wikimedia.org/w/index.php?title=Verksamhetsplan_2019/Tabell_%C3%B6ver_program,_strategi_och_m%C3%A5l&oldid=75471. - - """ - - if self._programs is not None: - return self._programs - - operational_plan_page = Page( - self._site, - self._make_year_title( - self._config["year_pages"]["operational_plan"]) - ) - if not operational_plan_page.exists(): - title = operational_plan_page.title() - raise PageMissingError(title) - - # Get table string. This assumes that it is the first table on - # the page. - table_string = str(mwp.parse( - operational_plan_page.text - ).filter_tags(matches=ftag('table'))[0]) - # Remove ref tags and links. - table_string = re.sub( - r"(.*?|\[\[.*?\||\]\])", - "", - table_string, - flags=re.S - ) - self._programs = [] - remaining_projects = list(self._projects.keys()) - # Split table on rows. - rows = table_string.split("|-") - for row in rows[1:]: - # Skip first rows; we don't need the headers. - if not row.rstrip("|}").strip(): - # This is just the end table row, skip it. - continue - # Split rows on pipes and remove formatting. - cells = list(filter(None, map( - lambda c: c.split("|")[-1].strip(), - re.split(r"[\|\n]\|", row) - ))) - if len(cells) == 3: - # Row includes program. - program_name, program_number = \ - re.match(r"(.*)\s+", cells[0]).groups() - self._programs.append({ - "number": program_number, - "name": program_name, - "strategies": [] - }) - if len(cells) >= 2: - # Row includes strategy, which is always in the cell - # second from the right. - strategy, strategy_number, strategy_short = \ - re.match( - r"(.*)\s*", - cells[-2] - ).groups() - self._programs[-1]["strategies"].append({ - "number": strategy_number, - "description": strategy, - "short_description": strategy_short, - "projects": [], - "goals": [] - }) - for project in self._get_projects_for_strategy( - strategy_number - ): - # Add projects for this strategy. - self._programs[-1]["strategies"][-1]["projects"].append( - project - ) - remaining_projects.remove(project) - # The rightmost cell always contains a goal. - goal = cells[-1] - self._programs[-1]["strategies"][-1]["goals"].append(goal) - if remaining_projects: - logging.warning( - "There were projects which could not be matched to programs, " - "these will be skipped from overview pages: '{}'".format( - ', '.join(remaining_projects) - ) - ) - return self._programs - - def _add_program_overview_year_page(self): - """Add a page with program overview. - - Uses several templates to build a table with information about - and status of goals and projects, ordered by program and - strategy. - - """ - - config = self._config["year_pages"]["program_overview"] - title = self._make_year_title( - self._config["year_pages"]["program_overview"]["title"] - ) - templates = config["templates"] - content_parameter = "" - try: - programs = self._get_programs() - except PageMissingError as error: - logging.error(f"Error when processing '{title}'.") - raise error - for p, program in enumerate(programs): - content_parameter += Template( - templates["program"], - True, - { - "program": program["name"], - "färg": config["colours"][p] - } - ).multiline_string() - content_parameter += "\n" - for strategy in program["strategies"]: - content_parameter += Template( - templates["strategy"], - True, - [strategy["description"]] - ).multiline_string() - content_parameter += "\n" - for goal in strategy["goals"]: - content_parameter += Template( - templates["goal"], - True, - [goal] - ).multiline_string() - content_parameter += "\n" - for project in strategy["projects"]: - content_parameter += Template( - templates["project"], - True, - [project] - ).multiline_string() - content_parameter += "\n" - self._add_page_from_template( - None, - title, - templates["page"], - { - "år": self._year, - "tabellinnehåll": content_parameter - } - ) + if program not in self._programs: + self._programs.append(program) def _add_year_categories(self): """Add category pages for a year. @@ -690,18 +529,14 @@ def _create_current_projects_template(self): ns=self._config["project_namespace"]) delimiter = "''' · '''" template_data = {} - try: - programs = self._get_programs() - except PageMissingError as error: - logging.error(f"Error when processing '{page_name}'.") - raise error - for program in programs: - projects = set() - for strategy in program.get('strategies'): - # projects sorted by id to get thematic grouping - projects.update(strategy.get("projects")) - template_data[program.get('name')] = delimiter.join( - [project_format.format(proj=self._projects[project]["sv"]) + for program in self._programs: + projects = [] + for _, project in self._projects.items(): + if project["program"] == program: + projects.append(project["sv"]) + + template_data[program] = delimiter.join( + [project_format.format(proj=project) for project in sorted(projects)]) template = Template("Aktuella projekt/layout") @@ -725,16 +560,11 @@ def _add_volunteer_tasks_page(self): config = self._config["year_pages"]["volunteer_tasks"] title = self._make_year_title(config["title"]) project_list_string = "" - try: - programs = self._get_programs() - except PageMissingError as error: - logging.error(f"Error when processing '{title}'.") - raise error - for program in programs: - project_list_string += "== {} ==\n".format(program["name"]) - for strategy in program["strategies"]: - for number in strategy["projects"]: - project_name = self._projects[number]["sv"] + for program_name in self._programs: + project_list_string += "== {} ==\n".format(program_name) + for _, project in self._projects.items(): + if project["program"] == program_name: + project_name = project["sv"] project_template = "{{" + \ ":Projekt:{}/Frivillig".format(project_name) + "}}\n" project_list_string += project_template