-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Automatically Generate Repository Git Page (#391)
* add git page action * Add scripts to create dot and png files for workflows * Add scripts to add png link to md * Add mkdocs.yml file * changed png link to be relative path instead of absolute. * Added script to add header to wdl markdown, and remove task info from wdl workflow markdown * edited wdlviz.py to add the option to name the output file by a specified name, edited create_wdl_visual.py to use wdl basename for dot file --------- Co-authored-by: bshifaw <bshifaw@broadinstitute.com>
- Loading branch information
Showing
7 changed files
with
655 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
name: Build and deploy site to GitHub Pages | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
workflow_dispatch: | ||
|
||
jobs: | ||
github-pages: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout Repository | ||
uses: actions/checkout@v3 | ||
|
||
- name: Cache node_modules | ||
uses: actions/cache@v3 | ||
with: | ||
path: "**/node_modules" | ||
key: ${{ runner.os }}-modules-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }} | ||
|
||
- name: Setup Python | ||
uses: actions/setup-python@v4 | ||
with: | ||
python-version: '3.10' | ||
|
||
- name: Install Requirements | ||
run: | | ||
python3 -m pip install --upgrade pip | ||
python3 -m pip install wdldoc~=1.7.0 | ||
python3 -m pip install mkdocs~=1.4.2 | ||
- name: Create Site Homepage | ||
run: | | ||
cp README.md docs/index.md && sed -i.bak 's/\.\/docs/\.\//g' docs/index.md | ||
- name: Build Docs | ||
run: | | ||
wdldoc --debug -o docs ./wdl/pipelines ./wdl/tasks | ||
python3 ./scripts/git_page/md_post_process.py --debug --md_dir ./docs | ||
python3 ./scripts/git_page/create_wdl_visual.py --debug ./wdl/pipelines ./wdl/tasks --output_path ./docs/dot | ||
python3 ./scripts/git_page/add_dot_link_to_md.py --md_dir ./docs/workflows --dot_dir ./docs/dot | ||
mkdocs build | ||
- name: Deploy | ||
uses: JamesIves/github-pages-deploy-action@releases/v4 | ||
with: | ||
BRANCH: gh-pages | ||
FOLDER: ./site |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
site_name: "Broad Institute Long Read Pipelines" | ||
repo_url: "https://github.com/broadinstitute/long-read-pipelines" | ||
|
||
site_description: "This repository contains pipelines for processing of long read data from PacBio and/or Oxford Nanopore platforms." | ||
site_author: "Broad Institute" | ||
copyright: "Copyright (c) 2023, Broad Institute" | ||
|
||
theme: readthedocs | ||
|
||
markdown_extensions: | ||
- def_list | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
import errno | ||
|
||
import os | ||
import argparse | ||
from pathlib import Path | ||
import logging | ||
|
||
import utility as util | ||
|
||
Logger = logging.getLogger(__name__) | ||
logging.basicConfig() | ||
|
||
|
||
def main(): | ||
parser = argparse.ArgumentParser( | ||
description="Add dot header and dot.png link to workflow markdown files" | ||
) | ||
parser.add_argument( | ||
"--md_dir", metavar="MARKDOWN_DIRECTORY", nargs='+', required=True, | ||
help="One of or more directories containing markdown " | ||
"documentation files for workflow" | ||
) | ||
parser.add_argument( | ||
"--dot_dir", metavar="DOT_DIRECTORY", nargs='+', required=True, | ||
help="One of or more directories containing dot.png " | ||
"files for workflows" | ||
) | ||
parser.add_argument("--debug", action="store_true", help="verbose logging") | ||
|
||
args = parser.parse_args() | ||
|
||
util.set_logging_level(args) | ||
|
||
md_dir_names = args.md_dir | ||
dot_png_dir_names = args.dot_dir | ||
|
||
dot_png_paths = get_all_dot_png_in_directory(dot_png_dir_names) | ||
|
||
if len(dot_png_paths) == 0: | ||
Logger.error(f'Could not find any dot.png files in {dot_png_dir_names}!') | ||
raise FileNotFoundError( | ||
errno.ENOENT, os.strerror(errno.ENOENT), dot_png_paths) | ||
|
||
for md_dir_name in md_dir_names: | ||
|
||
Logger.debug(f'Processing {md_dir_name}...') | ||
|
||
md_dir = util.get_absolute_path(md_dir_name) | ||
|
||
md_paths = util.get_all_files_with_extension(directory=md_dir, ext='md') | ||
|
||
for md in md_paths: | ||
Logger.debug(f'Processing {md}...') | ||
append_png_to_md(md_path=md, dot_paths=dot_png_paths) | ||
|
||
print("Done!") | ||
|
||
|
||
def append_png_to_md(md_path: str, dot_paths: list) -> None: | ||
""" | ||
Append dot.png link to markdown file | ||
@param md_path: path to markdown file | ||
@param dot_paths: list of paths to dot.png files | ||
@return: | ||
""" | ||
|
||
md_basename = util.get_basename(md_path) | ||
|
||
dot_png_matches = [] | ||
|
||
for dot_path in dot_paths: | ||
if md_basename == util.get_basename(dot_path): | ||
dot_png_matches.append(dot_path) | ||
|
||
if len(dot_png_matches) == 1: | ||
relative_d_p_path = get_relative_path(dot_png_matches[0], md_path) | ||
with open(md_path, 'a') as md_file: | ||
md_file.write(f'### Dot Diagram\n') | ||
md_file.write(f'![{md_basename}](../{relative_d_p_path})\n') | ||
elif len(dot_png_matches) > 1: | ||
Logger.error(f'Found more than one dot.png for {md_basename}!') | ||
|
||
|
||
def get_all_dot_png_in_directory(directory_names: list) -> list: | ||
""" | ||
Get all dot.png files in a list of directories | ||
@param directory_names: list of directories to search | ||
@return: list of paths to dot.png files | ||
""" | ||
|
||
dot_png_paths = [] | ||
for dot_png_dir_name in directory_names: | ||
dot_png_dir = util.get_absolute_path(dot_png_dir_name) | ||
|
||
dot_png_paths = ( | ||
dot_png_paths + util.get_all_files_with_extension( | ||
directory=dot_png_dir, ext='dot.png' | ||
) | ||
) | ||
return dot_png_paths | ||
|
||
|
||
def get_relative_path(path, relative_to_path) -> Path: | ||
""" | ||
Get the relative path for a path | ||
@param relative_to_path: Path to get relative path to | ||
@param path: Path to get relative path for | ||
@return: | ||
""" | ||
p = Path(path) | ||
parent_dir = Path(relative_to_path).parent | ||
return p.relative_to(parent_dir.parent) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
import argparse | ||
import logging | ||
from pathlib import Path | ||
|
||
import utility as util | ||
import wdlviz | ||
import WDL | ||
|
||
Logger = logging.getLogger(__name__) | ||
logging.basicConfig() | ||
|
||
|
||
def main(): | ||
# read command-line arguments | ||
parser = argparse.ArgumentParser( | ||
description="Visualize a WDL workflow using miniwdl and graphviz" | ||
) | ||
parser.add_argument( | ||
"wdl_dir", metavar="WDL_DIRECTORY", nargs='+', | ||
help="One of or more directories containing WDL workflow files" | ||
) | ||
parser.add_argument("--output_path", help="Path to output directory") | ||
parser.add_argument("--debug", action="store_true", help="verbose logging") | ||
|
||
args = parser.parse_args() | ||
|
||
util.set_logging_level(args) | ||
|
||
dir_names = args.wdl_dir | ||
output_path = str(util.get_absolute_path(args.output_path)) if args.output_path else None | ||
|
||
for dir_name in dir_names: | ||
|
||
Logger.debug(f'Processing {dir_name}...') | ||
|
||
wdl_dir = util.get_absolute_path(dir_name) | ||
|
||
wdl_paths = util.get_all_files_with_extension(directory=wdl_dir, ext='wdl') | ||
|
||
for wdl in wdl_paths: | ||
Logger.debug(f'Processing {wdl}...') | ||
|
||
if WDL.load(wdl).workflow: # check if wdl is a workflow | ||
run_wdlviz(wdl_path=wdl, output_path=output_path) | ||
else: | ||
Logger.debug(f"{wdl} is not a WDL workflow (does not contain a " | ||
"workflow block). Skipping...") | ||
|
||
|
||
def run_wdlviz( | ||
wdl_path: str, use_basename: bool = True, output_path: str = None | ||
) -> None: | ||
""" | ||
Run wdlviz on a WDL file to create dot and png files | ||
@param use_basename: name output files using the basename of the WDL file | ||
@param wdl_path: path to WDL file | ||
@param output_path: path to output directory | ||
@return: | ||
""" | ||
wdlviz_args = [wdl_path] | ||
|
||
if output_path is not None: | ||
wdlviz_args.append("--output_path") | ||
wdlviz_args.append(output_path) | ||
if use_basename: | ||
wdlviz_args.append("--dot-basename") | ||
wdlviz_args.append(Path(wdl_path).stem) | ||
|
||
Logger.debug(f'Running wdlviz with args: {wdlviz_args}...') | ||
wdlviz.main(args=wdlviz_args) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() | ||
|
Oops, something went wrong.