From 652ce359d82a51aeb3155b62170f6ff547670ac7 Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Tue, 21 May 2024 14:42:46 +0200 Subject: [PATCH 1/6] Uploading the branch into the GH --- debugme.sh | 0 src/containers/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 debugme.sh create mode 100644 src/containers/__init__.py diff --git a/debugme.sh b/debugme.sh new file mode 100644 index 0000000..e69de29 diff --git a/src/containers/__init__.py b/src/containers/__init__.py new file mode 100644 index 0000000..e69de29 From f643bcb89ca2d3ef1d922e8933d68a248626f57f Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Wed, 22 May 2024 10:19:23 +0200 Subject: [PATCH 2/6] Running script initial commit. --- notes.txt | 6 ------ src/clean_env_before_mining.py | 6 ++---- src/controller.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 notes.txt diff --git a/notes.txt b/notes.txt deleted file mode 100644 index 6a1ad23..0000000 --- a/notes.txt +++ /dev/null @@ -1,6 +0,0 @@ -python3 scripts/controller.py - --github-token "${githubToken}" \ - --project-state-mining "${projectStateMining}" \ - --projects-title-filter "${projectsTitleFilter}" \ - --milestones-as-chapters "${milestonesAsChapters}" \ - --repositories '${repositories}'` diff --git a/src/clean_env_before_mining.py b/src/clean_env_before_mining.py index 7c10332..9246cec 100644 --- a/src/clean_env_before_mining.py +++ b/src/clean_env_before_mining.py @@ -37,7 +37,7 @@ def clean_directory_content(script_dir: str, directory: str) -> None: shutil.rmtree(directory_path) -def clean_environment(env_vars): +def clean_environment(): print("Cleaning of env for Living Documentation started") # Get the directory of the current script @@ -54,8 +54,6 @@ def clean_environment(env_vars): if __name__ == "__main__": - import sys - env_vars = sys.argv[1:] - clean_environment(env_vars) + clean_environment() # TODO - move dir variables definition to the controlling script - add them to the env_vars # - to able to provide them to all rest scripts #Issue: https://github.com/AbsaOSS/living-doc-generator/issues/4 diff --git a/src/controller.py b/src/controller.py index 52dd97a..9073921 100644 --- a/src/controller.py +++ b/src/controller.py @@ -50,7 +50,7 @@ def main(): print("Data mining for Living Documentation") - clean_environment(env_vars) + clean_environment() # Data mine GitHub features from repository run_script('github_query_issues.py', env_vars) From ee9ca7e2cfa1257afe3032920e332057a1d6fdba Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Wed, 22 May 2024 10:20:56 +0200 Subject: [PATCH 3/6] Running script initial commit. --- dist/clean_env_before_mining.py | 12 +++++++++--- dist/controller.py | 10 +++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/dist/clean_env_before_mining.py b/dist/clean_env_before_mining.py index 0b30b1e..9246cec 100644 --- a/dist/clean_env_before_mining.py +++ b/dist/clean_env_before_mining.py @@ -37,8 +37,8 @@ def clean_directory_content(script_dir: str, directory: str) -> None: shutil.rmtree(directory_path) -if __name__ == "__main__": - print("Data mining for Living Documentation started") +def clean_environment(): + print("Cleaning of env for Living Documentation started") # Get the directory of the current script script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -50,4 +50,10 @@ def clean_directory_content(script_dir: str, directory: str) -> None: # Clean the output directory clean_directory_content(script_dir, MARKDOWN_PAGE_DIRECTORY) - print("Data mining for Living Documentation ended") + print("Cleaning of env for Living Documentation ended") + + +if __name__ == "__main__": + clean_environment() + # TODO - move dir variables definition to the controlling script - add them to the env_vars + # - to able to provide them to all rest scripts #Issue: https://github.com/AbsaOSS/living-doc-generator/issues/4 diff --git a/dist/controller.py b/dist/controller.py index 56381d3..9073921 100644 --- a/dist/controller.py +++ b/dist/controller.py @@ -2,7 +2,7 @@ import subprocess import argparse import sys -import re +from clean_env_before_mining import clean_environment def extract_args(): @@ -48,24 +48,20 @@ def main(): print("Extracting arguments from command line.") env_vars = extract_args() - # Clean environment before mining print("Data mining for Living Documentation") - run_script('clean_env_before_mining.py', env_vars) + + clean_environment() # Data mine GitHub features from repository - print("Downloading issues from GitHub") run_script('github_query_issues.py', env_vars) # Data mine GitHub project's state - print("Downloading project's state") run_script('github_query_project_state.py', env_vars) # Consolidate all feature data together - print("Consolidating mined feature data") run_script('consolidate_feature_data.py', env_vars) # Generate markdown pages - print("Converting features to markdown pages") run_script('convert_features_to_pages.py', env_vars) From cfd3fd3ca457e96f2b434240ad6391f0c17a524c Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Thu, 23 May 2024 13:12:11 +0200 Subject: [PATCH 4/6] README file updated with steps about running the scripts locally --- README.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ae2ae11..f341198 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ - [License Information](#license-information) - [Contact or Support Information](#contact-or-support-information) -A tool designed to data-mine GitHub repositories for issues containing project documentation (e.g. tagged with feature-related labels). This tool automatically generates comprehensive living documentation in markdown format, providing detailed feature overview pages and in-depth feature descriptions. +A tool designed to data-mine GitHub repositories for issues containing project documentation (e.g. tagged with feature-related labels). This tool automatically generates comprehensive living documentation in Markdown format, providing detailed feature overview pages and in-depth feature descriptions. ## Motivation Addresses the need for continuously updated documentation accessible to all team members and stakeholders. Achieves this by extracting information directly from GitHub issues and integrating this functionality to deliver real-time, markdown-formatted output. Ensures everyone has the most current project details, fostering better communication and efficiency throughout development. @@ -211,6 +211,71 @@ source venv/bin/activate pip install -r requirements.txt ``` +## Run scripts locally +If you need to run the scripts locally, follow these steps: + +### Create the shell script +Create the shell file in the root directory. We will use `run_script.sh`. +```shell +touch run_script.sh +``` +Add the shebang line at the top of the sh script file. +``` +#!/bin/sh +``` + +### Set the environment variables +Set the configuration environment variables in the shell script following the structure below. +Also make sure that the GITHUB_TOKEN is configured in your environment variables. +``` +export GITHUB_TOKEN=$(printenv GITHUB_TOKEN) +export PROJECT_STATE_MINING="true" +export PROJECTS_TITLE_FILTER="[]" +export MILESTONES_AS_CHAPTERS="true" +export REPOSITORIES='[ + { + "orgName": "OrgName", + "repoName": "example-project", + "queryLabels": ["feature", "bug"] + } + ]' +``` + +### Running the GH action locally +For running the whole GitHub action, add the following commands to the shell script: +``` +cd src || exit 1 + +python3 controller.py --github-token "$GITHUB_TOKEN" \ + --project-state-mining "$PROJECT_STATE_MINING" \ + --projects-title-filter "$PROJECTS_TITLE_FILTER" \ + --milestones-as-chapters "$MILESTONES_AS_CHAPTERS" \ + --repositories "$REPOSITORIES" + +cd .. || exit 1 +``` + +### Running single script locally +For running just one script at the time, add the following commands to the shell script: +``` +cd src || exit 1 + +python3 github_query_project_state.py + +cd .. || exit 1 +``` + +### Make the Script Executable +From the terminal that is in the root of this project, make the script executable: +```shell +chmod +x run_script.sh +``` + +### Run the Script +```shell +./run_script.sh +``` + ## Run unit test TODO - check this chapter and update by latest state ### Launch unit tests From 8c5fcc13986bbbef84671536aea5de2c78ad6470 Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Thu, 23 May 2024 14:29:41 +0200 Subject: [PATCH 5/6] Script running logic from one place fully integrated --- .gitignore | 13 +++++++++++++ dist/clean_env_before_mining.py | 23 ++++++++++------------- dist/controller.py | 32 +++++++++++++++++++------------- src/clean_env_before_mining.py | 23 ++++++++++------------- src/controller.py | 32 +++++++++++++++++++------------- 5 files changed, 71 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 3474bb8..ce31edc 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,16 @@ cython_debug/ /.idea/ node_modules /package-lock.json + +# Generated outputs +dist/data/ +dist/output/ +src/data/ +src/output/ + +# Debugging files +debugme_script.sh +debugme.sh + + + diff --git a/dist/clean_env_before_mining.py b/dist/clean_env_before_mining.py index 9246cec..8b7f29d 100644 --- a/dist/clean_env_before_mining.py +++ b/dist/clean_env_before_mining.py @@ -11,12 +11,6 @@ import shutil -# Directories used in the project -FETCH_DIRECTORY = "data/fetched_data" -CONSOLIDATION_DIRECTORY = "data/consolidation_data" -MARKDOWN_PAGE_DIRECTORY = "output/markdown_pages" - - def clean_directory_content(script_dir: str, directory: str) -> None: """ Deletes all content from the specified directory. @@ -38,22 +32,25 @@ def clean_directory_content(script_dir: str, directory: str) -> None: def clean_environment(): - print("Cleaning of env for Living Documentation started") + print("Cleaning environment for the Living Doc Generator") # Get the directory of the current script script_dir = os.path.dirname(os.path.abspath(__file__)) + # Get the directory variables from the environment variables + fetch_directory = os.environ['FETCH_DIRECTORY'] + consolidation_directory = os.environ['CONSOLIDATION_DIRECTORY'] + markdown_page_directory = os.environ['MARKDOWN_PAGE_DIRECTORY'] + # Clean the fetched data directories - clean_directory_content(script_dir, FETCH_DIRECTORY) - clean_directory_content(script_dir, CONSOLIDATION_DIRECTORY) + clean_directory_content(script_dir, fetch_directory) + clean_directory_content(script_dir, consolidation_directory) # Clean the output directory - clean_directory_content(script_dir, MARKDOWN_PAGE_DIRECTORY) + clean_directory_content(script_dir, markdown_page_directory) print("Cleaning of env for Living Documentation ended") if __name__ == "__main__": - clean_environment() - # TODO - move dir variables definition to the controlling script - add them to the env_vars - # - to able to provide them to all rest scripts #Issue: https://github.com/AbsaOSS/living-doc-generator/issues/4 + clean_environment() \ No newline at end of file diff --git a/dist/controller.py b/dist/controller.py index 9073921..0a33fa4 100644 --- a/dist/controller.py +++ b/dist/controller.py @@ -22,20 +22,19 @@ def extract_args(): 'PROJECT_STATE_MINING': args.project_state_mining, 'PROJECTS_TITLE_FILTER': args.projects_title_filter, 'MILESTONES_AS_CHAPTERS': args.milestones_as_chapters, - 'REPOSITORIES': args.repositories + 'REPOSITORIES': args.repositories, + 'FETCH_DIRECTORY': "src/data/fetched_data", + 'CONSOLIDATION_DIRECTORY': "src/data/consolidation_data", + 'MARKDOWN_PAGE_DIRECTORY': "src/output/markdown_pages" } return env_vars -def run_script(script_name, env_vars): +def run_script(script_name, env): """ Helper function to run a Python script with environment variables using subprocess """ try: - # Setting up the environment for subprocess - env = os.environ.copy() - env.update(env_vars) - - # Running the script with updated environment + # Running the python script with given environment variables result = subprocess.run(['python3', script_name], env=env, text=True, capture_output=True, check=True) print(f"Output from {script_name}: {result.stdout}") @@ -48,21 +47,28 @@ def main(): print("Extracting arguments from command line.") env_vars = extract_args() - print("Data mining for Living Documentation") + # Create a local copy of the current environment variables + local_env = os.environ.copy() + + # Add the script-specific environment variables to the local copy + local_env.update(env_vars) + + print("Starting the Living Documentation Generator - mining phase") - clean_environment() + # Clean the environment before mining + run_script('clean_env_before_mining.py', local_env) # Data mine GitHub features from repository - run_script('github_query_issues.py', env_vars) + run_script('github_query_issues.py', local_env) # Data mine GitHub project's state - run_script('github_query_project_state.py', env_vars) + run_script('github_query_project_state.py', local_env) # Consolidate all feature data together - run_script('consolidate_feature_data.py', env_vars) + run_script('consolidate_feature_data.py', local_env) # Generate markdown pages - run_script('convert_features_to_pages.py', env_vars) + run_script('convert_features_to_pages.py', local_env) if __name__ == '__main__': diff --git a/src/clean_env_before_mining.py b/src/clean_env_before_mining.py index 9246cec..8b7f29d 100644 --- a/src/clean_env_before_mining.py +++ b/src/clean_env_before_mining.py @@ -11,12 +11,6 @@ import shutil -# Directories used in the project -FETCH_DIRECTORY = "data/fetched_data" -CONSOLIDATION_DIRECTORY = "data/consolidation_data" -MARKDOWN_PAGE_DIRECTORY = "output/markdown_pages" - - def clean_directory_content(script_dir: str, directory: str) -> None: """ Deletes all content from the specified directory. @@ -38,22 +32,25 @@ def clean_directory_content(script_dir: str, directory: str) -> None: def clean_environment(): - print("Cleaning of env for Living Documentation started") + print("Cleaning environment for the Living Doc Generator") # Get the directory of the current script script_dir = os.path.dirname(os.path.abspath(__file__)) + # Get the directory variables from the environment variables + fetch_directory = os.environ['FETCH_DIRECTORY'] + consolidation_directory = os.environ['CONSOLIDATION_DIRECTORY'] + markdown_page_directory = os.environ['MARKDOWN_PAGE_DIRECTORY'] + # Clean the fetched data directories - clean_directory_content(script_dir, FETCH_DIRECTORY) - clean_directory_content(script_dir, CONSOLIDATION_DIRECTORY) + clean_directory_content(script_dir, fetch_directory) + clean_directory_content(script_dir, consolidation_directory) # Clean the output directory - clean_directory_content(script_dir, MARKDOWN_PAGE_DIRECTORY) + clean_directory_content(script_dir, markdown_page_directory) print("Cleaning of env for Living Documentation ended") if __name__ == "__main__": - clean_environment() - # TODO - move dir variables definition to the controlling script - add them to the env_vars - # - to able to provide them to all rest scripts #Issue: https://github.com/AbsaOSS/living-doc-generator/issues/4 + clean_environment() \ No newline at end of file diff --git a/src/controller.py b/src/controller.py index 9073921..0a33fa4 100644 --- a/src/controller.py +++ b/src/controller.py @@ -22,20 +22,19 @@ def extract_args(): 'PROJECT_STATE_MINING': args.project_state_mining, 'PROJECTS_TITLE_FILTER': args.projects_title_filter, 'MILESTONES_AS_CHAPTERS': args.milestones_as_chapters, - 'REPOSITORIES': args.repositories + 'REPOSITORIES': args.repositories, + 'FETCH_DIRECTORY': "src/data/fetched_data", + 'CONSOLIDATION_DIRECTORY': "src/data/consolidation_data", + 'MARKDOWN_PAGE_DIRECTORY': "src/output/markdown_pages" } return env_vars -def run_script(script_name, env_vars): +def run_script(script_name, env): """ Helper function to run a Python script with environment variables using subprocess """ try: - # Setting up the environment for subprocess - env = os.environ.copy() - env.update(env_vars) - - # Running the script with updated environment + # Running the python script with given environment variables result = subprocess.run(['python3', script_name], env=env, text=True, capture_output=True, check=True) print(f"Output from {script_name}: {result.stdout}") @@ -48,21 +47,28 @@ def main(): print("Extracting arguments from command line.") env_vars = extract_args() - print("Data mining for Living Documentation") + # Create a local copy of the current environment variables + local_env = os.environ.copy() + + # Add the script-specific environment variables to the local copy + local_env.update(env_vars) + + print("Starting the Living Documentation Generator - mining phase") - clean_environment() + # Clean the environment before mining + run_script('clean_env_before_mining.py', local_env) # Data mine GitHub features from repository - run_script('github_query_issues.py', env_vars) + run_script('github_query_issues.py', local_env) # Data mine GitHub project's state - run_script('github_query_project_state.py', env_vars) + run_script('github_query_project_state.py', local_env) # Consolidate all feature data together - run_script('consolidate_feature_data.py', env_vars) + run_script('consolidate_feature_data.py', local_env) # Generate markdown pages - run_script('convert_features_to_pages.py', env_vars) + run_script('convert_features_to_pages.py', local_env) if __name__ == '__main__': From 7ec157261c29a87d71919089c2f852834b389697 Mon Sep 17 00:00:00 2001 From: MobiTikula Date: Mon, 27 May 2024 09:15:14 +0200 Subject: [PATCH 6/6] Fixing importing libraries --- dist/controller.py | 1 - src/controller.py | 1 - 2 files changed, 2 deletions(-) diff --git a/dist/controller.py b/dist/controller.py index 0a33fa4..92acad4 100644 --- a/dist/controller.py +++ b/dist/controller.py @@ -2,7 +2,6 @@ import subprocess import argparse import sys -from clean_env_before_mining import clean_environment def extract_args(): diff --git a/src/controller.py b/src/controller.py index 0a33fa4..92acad4 100644 --- a/src/controller.py +++ b/src/controller.py @@ -2,7 +2,6 @@ import subprocess import argparse import sys -from clean_env_before_mining import clean_environment def extract_args():