Feature/126 install docs

.github/jobs/build_documentation.sh

@@ -8,6 +8,7 @@ DOCS_DIR=${GITHUB_WORKSPACE}/docs
 # run Make to build the documentation and return to previous directory
 cd ${DOCS_DIR}
 make clean html
+status=$?
 cd -
 
 # copy HTML output into directory to create an artifact
@@ -18,7 +19,10 @@ cp -r ${DOCS_DIR}/_build/html/* artifact/documentation
 # Copy it into the artifact and documeentation directories
 # so it will be available in the artifacts
 warning_file=${DOCS_DIR}/_build/warnings.log
-if [ -s $warning_file ]; then
+if [[ $status != 0 || -s $warning_file ]]; then
+  if [ $status != 0 ]; then
+    echo "ERROR: 'make clean html' failed with status $status."
+  fi
   cp -r ${DOCS_DIR}/_build/warnings.log artifact/doc_warnings.log
   cp artifact/doc_warnings.log artifact/documentation
   exit 1

.github/workflows/documentation.yml

@@ -6,8 +6,9 @@ on:
   push:
     branches:
       - develop
-      - feature*
-      - bugfix*
+      - 'main**'
+      - 'feature**'
+      - 'bugfix**'
     paths: 
       - docs/**
   pull_request:
@@ -25,8 +26,8 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade python-dateutil requests sphinx \
-          sphinx-panels sphinx_rtd_theme
+          python -m pip install --upgrade sphinx sphinx-gallery sphinx_rtd_theme
+          python -m pip install python-dateutil requests Pillow
       - name: Build docs
         run: ./.github/jobs/build_documentation.sh   
       - uses: actions/upload-artifact@v3
@@ -40,3 +41,4 @@ jobs:
           name: documentation_warnings.log
           path: artifact/doc_warnings.log
           if-no-files-found: ignore
+

docs/Users_Guide/installation.rst

@@ -7,35 +7,243 @@ System Installation
 Introduction
 ============
 
-The WRF Cloud framework can be installed in any AWS account via CloudShell.
+The WRF Cloud framework can be installed in any AWS account via `CloudShell <https://aws.amazon.com/cloudshell>`_.
+
+If you encounter issues related to your AWS account, please work with your AWS account administrator. For issues related to the WRF Cloud installation, please post to `WRF Cloud GitHub Discussions <https://github.com/NCAR/wrfcloud/discussions>`_.
+
+.. _permissions:
+
+Required Permissions
+====================
+
+The user installing WRF Cloud must have sufficient administrative permissions within their AWS account. It is recommended to install with **AdministratorAccess**, **SystemAdministrator**, or root privileges. Please work with your AWS account administrator to secure the required permissions or find another user with sufficient permissions to perform the installation.
+
+Check existing permissions:
+
+* In the **AWS Management Console**, use the top-level search bar to find and launch the AWS **IAM** (Identity and Access Management) Service.
+* If you are unable to launch the AWS **IAM** Service, you do not have sufficient permissions.
+* In **Access management > Users**, find and select your user name, and inspect the **Permissions policies**.
+* Ensure that you have **AdministratorAccess**, **SystemAdministrator**, or higher permissions.
+
+If any of the following installation steps fail due to insufficient permissions, please contact your AWS account administrator for assistance.
+
+.. _prerequisites:
 
 Prerequisites
 =============
 
-* AWS Account
-* Domain name in Route 53
-* AWS account limit for "Running On-Demand All HPC instances" in us-east-2 must be 96 vCPUs or higher
-* Simple Email Service (SES) must be out of the sandbox in us-east-2
+The are 3 prerequisites for installing WRF Cloud in an AWS account:
+
+* **Web Domain**: This is the web domain (hosted zone) where the WRF Cloud system is accessed. AWS's `Route 53 <https://aws.amazon.com/route53>`_ Domain Naming Service (DNS) is used to create and host the domain for the system. This service has an annual cost associated with it.
+* **Adequate Instance Limits**: On-demand HPC instance limits must be set to a minimum of 96 vCPUs. By default, the AWS account may be less than this. A request should be made to increase this limit for full functionality of the system.
+* **Enable Simple Email Service (SES)**: This facilitates AWS emailing within the system. By default the SES is limited to testing with verified identities. A request must be made to lift these limits and make it a production service.
+
+The procedures for ensuring the prerequisites are fulfilled are described below.
+
+1. Login to the `AWS Management Console <https://aws.amazon.com/console>`_ for your account as a user with sufficient permissions (see :numref:`permissions`).
+
+2. Check `Route 53 <https://aws.amazon.com/route53>`_ domain name.
+
+   * Use the top-level search bar to find and launch the AWS **Route 53** Service.
+   * Check for at least one **Hosted zone** or create a new one with an annual fee.
+
+3. Check account limits.
+
+   * Use the top-level search bar to find and launch the AWS **EC2** Service.
+   * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+   * Select **Limits** from the left navigation menu and search for **HPC** to find **Running On-Demand HPC instances**.
+   * If the **Current limit** is less than **96 vCPUs**, select the radio button and click **Request limit increase**.
+   * Check your email for followup and confirmation from AWS support.
+
+4. Check Simple Email Service (SES).
+
+   * Use the top-level search bar to find and launch the AWS **SES** Service.
+   * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+   * If you registered a new **Route 53** domain above, you'll need to create a new verified identity.
+
+      * Select **Configuration > Verified identities** from the left navigation menu.
+      * If the newly added domain is not listed, select the **Create identity** button, select the **Domain** radio button, and enter the newly added domain.
+      * Check your email for followup and confirmation from AWS support.
+      * Confirm that the newly added domain appears in the **Verified indentities** list.
+
+   * If you see a banner message warning that **Your Amazon SES account is in the sandbox in US East (Ohio)**, contact AWS to **Request production access**.
+
+      * Prior to requesting production access, be sure that you have at least one verified identity (see above).
+      * Check your email for followup and confirmation from AWS support.
+      * If AWS requires further justification for this change, note that this email service is needed by WRF Cloud to send users emails when WRF jobs finish running and to facilitate password recovery. We also recommend that your AWS account administrators subscribe to an SNS topic that is configured to accept bounce and complaint messages.
+
+.. _install:
+
+Procedures to Install
+=====================
+
+Once the AWS account prerequisites are satisfied (see :numref:`prerequisites`), follow these instructions to install WRF Cloud.
+
+1. Login to the `AWS Management Console <https://aws.amazon.com/console>`_ for your account as a user with sufficient permissions (see :numref:`permissions`).
+
+2. Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+
+3. Click the **CloudShell** button (|cloudshell_16x16|) at the top of the screen to launch a terminal window.
+
+.. |cloudshell_16x16| image:: figure/cloudshell_icon.png
+               :alt: CloudSell_Icon
+               :height: 16px
+               :width: 16px
+
+4. Define the GitHub branch name (e.g. develop) or release name (e.g. v1.0) to be installed.
+
+  .. code-block:: ini
+
+    export GITHUB_NAME=develop
+
+5. Copy and paste the following into the CloudShell terminal:
+
+  .. code-block:: ini
+
+    git clone --branch ${GITHUB_NAME} https://github.com/NCAR/wrfcloud
+    ./wrfcloud/install_bootstrap.sh ${GITHUB_NAME}
+
+This bootstrap script takes about 25 minutes to run. It is followed by a series of interactive questions that must be completed prior to WRF Cloud being installed. For each question, the default option (if applicable) is provided in square braces. Simply hit enter to accept the default or modify the setting as needed. These questions include:
+
+  * Which domain name would you like to use? [`Route 53 <https://aws.amazon.com/route53>`_ domain(s) from :numref:`prerequisites`]
+
+     * Note: This is used for {DOMAIN} in the following questions.
+
+  * Enter host name for web application: [e.g. app.{DOMAIN}]
+  * Enter host name for REST API: [e.g. api.{DOMAIN}]
+  * Enter host name for websocket API: [e.g. ws.{DOMAIN}]
+  * Enter administrator's full name:
+  * Enter email address for application administrator:
+  * Enter administrator's new password:
+  * Do you want to install example model configurations? [*Recommend Yes*]
+  * Do you want to upload an SSH public key for an admin? [*Recommend Yes*]
+  * Paste your public key, often found at ${HOME}/.ssh/id_rsa.pub:
+
+    * Copy and paste the output of this command into the CloudShell terminal:
+
+      .. code-block:: ini
+
+        cat ${HOME}/.ssh/id_rsa.pub
+
+      If this file does not exist, refer to `this documentation <https://git-scm.com/book/en/v2/Git-on-the-Server-Generating-Your-SSH-Public-Key>`_ to generate a public key.
+
+6. After completing these steps, check the email address provided above to verify it. It takes approximately 10-20 minutes for the website to become available.
+
+7. Monitor the installation progress:
+
+  * Launch a new `AWS Management Console <https://aws.amazon.com/console>`_ window, and use the top-level search bar to find and launch the AWS **CloudFormation** Service.
+  * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+  * Select **Stacks** from the left navigation menu.
+  * Click an item (**WrfIntelImageBuilder**, **WrfCloudApiData**, or **WrfCloudWebApp**) listed in the **Stack name** column and select the **Events** tab to monitor progress.
+
+8. When installation completes, a log message in the CloudShell terminal directs you to the newly created WRF Cloud URL.
+
+  .. code-block:: ini
+
+    WRF Cloud installation is complete.
+    Open your browser to https://app.{DOMAIN}
+
+9. Confirm that you can login.
+
+  * Open a web browser and clear the cache to ensure a clean test environment.
+  * Navigate to https://app.{DOMAIN}.
+  * If directed to the **WRF Cloud Login** page, use the administrator email address and password defined above to login and proceed to the next step.
+  * If, however, your browser downloads a file named **download** rather than allowing you to login, follow the steps outlined below to fix this behavior.
+
+    a. In the CloudShell terminal, check that temporary build directory still exists. It should remain for a while after install but will eventually be scrubbed from **/tmp**.
+
+    .. code-block:: ini
+
+       ls /tmp/wrfcloud-build-*
+
+    b. If it exists, run the following commands to update the WRF Cloud web files:
+
+    .. code-block:: ini
+
+      cd /tmp/wrfcloud-build-*
+      bucket=`aws s3 ls | sed -r 's/ /\n/g' | grep wrfcloud`
+      find web -type f -exec aws s3 cp {} s3://${bucket}/\{\} \;
+
+    c. Create an AWS **CloudFront** invalidation.
+
+      * Use the top-level search bar to find and launch the AWS **CloudFront** Service.
+      * In **Distributions**, click on the **ID** for the line whose **Description** is **wrfcloud production**.
+      * Select the **Invalidations** tab and click **Create Invalidation**.
+      * In the **Add object paths** text box, type **/\***, and click **Create Invalidation**.
+
+    d. In your browser window, clear your cache and navigate to https://app.{DOMAIN} again. If the **download** problem persists, repeats steps (a)-(c) until it stops. Remember to *clear your browser cache* before checking the URL.
+
+10. Confirm that the WRF Amazon Machine Image (AMI) has finished building.
+
+  * Use the top-level search bar to find and launch the AWS **EC2** Service.
+  * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+  * Select **Images > AMIs** from the left navigation menu and search for **wrf**.
+  * Confirm that an AMI that includes **wrf** in the **AMI name** column.
+  * Note that it may take several hours for this AMI to finish building. While you can add new users and configurations via the WRF Cloud user interface, you will not be able to actually launch a run until the WRF AMI is available.
+
+11. Afer completing these installation steps, proceed to System Administration (:numref:`administration`).
+
+.. _uninstall:
+
+Procedures to Uninstall
+=======================
+
+The steps for uninstalling WRF Cloud from an AWS account are described below.
+
+1. Login to the `AWS Management Console <https://aws.amazon.com/console>`_ for your account as a user with sufficient permissions (see :numref:`permissions`).
+
+2. Remove data from S3.
+
+  * Use the top-level search bar to find and launch the AWS **S3** Service.
+  * Select **Buckets** from the left navigation menu and search for **wrfcloud**.
+  * Select the **wrfcloud-XXXXXXX** S3 bucket and click **Empty** to remove its contents.
+
+    * Do not **Delete** this bucket, as CloudFormation will handle that step.
+
+3. Remove IAM policy.
+
+  * Use the top-level search bar to find and launch the AWS **IAM** Service.
+  * Select **Access Management > Policies** from the left navigation menu and search for **wrfcloud**.
+  * Click the **wrfcloud_parallelcluster** radio button and select **Actions > Delete**.
+
+    * Any other **wrfcloud** policies do not need to be removed.
+
+4. Modify EC2 settings.
+
+  * Use the top-level search bar to find and launch the AWS **EC2** Service.
+  * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+  * Remove the wrfcloud Amazon Machine Image (AMI).
+
+    * Select **Images > AMIs** from the left navigation menu and search for **wrf**.
+    * Make note of the **AMI ID** for the **wrf** AMI.
+    * Select that AMI followed by **Actions > Deregsiter AMI**.
+
+  * Remove any AMI Snapshots.
+
+    * Select **Elastic Block Store > Snapshots** from the left navigation menu.
+    * Select any entries for which the **AMI ID** noted above appears in the **Description** column.
+    * Select **Actions > Delete snapshot** to remove them.
+
+  * Remove the SSH key.
+
+    * Select **Network & Security > Key Pairs** from the left navigation menu.
+    * Select the **wrfcloud-amdmin** key pair followed by **Actions > Delete**.
 
-Procedures for initial installation
-===================================
+5. Modify SES settings.
 
-From the AWS web console, switch the region to us-east-2 (Ohio), then click on the CloudShell icon at the top of the screen to start CloudShell.
+  * Use the top-level search bar to find and launch the AWS **SES** Service.
+  * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+  * Select **Configuration > Verified Identites** from the left navigation menu.
+  * Select and **Delete** the email identity for your administrator's email address.
 
-From CloudShell:
+6. Delete CloudFormation stacks.
 
-* git clone --branch feature/auto-install https://github.com/NCAR/wrfcloud
-* ./wrfcloud/install_bootstrap.sh
-[The bootstrap takes about 25 minutes to complete and is followed by some questions before WRF Cloud is installed in your account.]
+  * Use the top-level search bar to find and launch the AWS **CloudFormation** Service.
+  * Select the **US East (Ohio) / us-east-2** region from the top-right dropdown navigation.
+  * Select **Stacks** from the left navigation menu.
+  * Select and **Delete** each of the following stacks: **WrfIntelImageBuilder**, **WrfCloudWebApp**, and **WrfCloudApiData**
 
-Procedures to uninstall the application
-=======================================
+    * Wait for the deletions to complete.
+    * If a delete fails, delete it again, and **DO NOT retain** any of optional resources.
 
-* Remove data from S3 bucket named wrfcloud-XXXXXXX (do not remove the bucket as CloudFormation will handle this)
-* Remove the wrfcloud_parallelcluster IAM policy
-* Remove the AMI from the EC2 service (be sure to deregister the AMI and delete the snapshots!)
-* Remove the SSH key from the EC2 service named wrfcloud-admin
-* Remove the SES email identity for your admin's email address
-* Delete CloudFormation stacks: WrfIntelImageBuilder, WrfCloudWebApp
-* Wait for WrfCloudWebApp stack to finish deleting.  If a delete fails, delete it again, and DO NOT retain the resources.
-* Delete WrfCloudApiData, WrfCloudWebCertificate (us-east-1)
+You have now finished uninstalling WRF Cloud from your AWS account.

docs/conf.py

@@ -32,8 +32,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-# Adding 'sphinx_panels' to use drop-down menus in appendixA. 
-extensions = ['sphinx.ext.autodoc','sphinx.ext.intersphinx','sphinx_panels',]
+extensions = ['sphinx.ext.autodoc','sphinx.ext.intersphinx',]
 
 # settings for ReadTheDocs PDF creation
 latex_engine = 'pdflatex'

docs/index.rst

@@ -13,7 +13,7 @@ The WRF Cloud framework is the realization of a idea years in the making to leve
 Acronyms
 --------
 
-* **MMM***  - Mesoscale and Microscale Meteorology Lab
+* **MMM**  - Mesoscale and Microscale Meteorology Lab
 * **NCAR**  - National Center for Atmospheric Research
 * **RAL**   - Research Applications Laboratory
 * **UCAR**  - University Corporation for Atmospheric Research

docs/requirements.txt

@@ -1,3 +1,3 @@
-sphinx-gallery
-sphinxcontrib-bibtex
-sphinx-panels
+sphinx-gallery==0.11.1
+sphinx==5.3.0
+sphinx-rtd-theme==1.1.1

install_bootstrap.sh

@@ -5,11 +5,18 @@
 # Post-condition: WRF Cloud will be installed in the environment and build artifacts will be available
 function main()
 {
+  ### Check for optional branch name ###
+  if [ $# -gt 0 ]; then
+    export GIT_CLONE_OPTS="--branch ${1}"
+  else
+    export GIT_CLONE_OPTS=""
+  fi
+
   ### Setup Build Directory
   time=$(date +%Y%m%d_%H%M%S)
   export build_dir="/tmp/wrfcloud-build-${time}"
   mkdir -p "${build_dir}" && cd "${build_dir}"
-  git clone --branch feature/auto-install https://github.com/NCAR/wrfcloud
+  git clone ${GIT_CLONE_OPTS} https://github.com/NCAR/wrfcloud
 
   ### Configure CloudShell Environment
   install_os_packages
@@ -91,7 +98,7 @@ function create_wrfcloud_lambda_layer()
   cd ../../
   ln -s ~/.nvm/versions/node/v16.19.0 $(pwd)/node
   rm -f ~/.nvm/versions/node/v16.19.0/v16.19.0
-  zip -r "${build_dir}/lambda_layer.zip" python/lib node
+  zip -r "${build_dir}/lambda_layer.zip" python/lib node/bin node/include node/share node/lib/node_modules/corepack node/lib/node_modules/npm
 }
 
 # Create a zip file for the lambda function
@@ -123,4 +130,4 @@ function install_wrfcloud()
   sudo /opt/python/bin/python3 -m pip install .
 }
 
-main
+main $@

python/src/setup.py

@@ -44,6 +44,7 @@
             'setup/aws/cf_wrfcloud_data.yaml',
             'setup/aws/cf_wrfcloud_certificate.yaml',
             'setup/aws/cf_wrfcloud_webapp.yaml',
+            'setup/resources/caribbean_6km_config.yaml',
             'setup/aws/wrfcloud_cluster_policy.json',
             'runtime/configurations/test/namelist.*',
             'runtime/resources/*.yaml',

python/src/wrfcloud/aws/imagebuilder.py

@@ -26,6 +26,10 @@ def __init__(self, stack_name: str = None, cf_file: str = None, image_name: str
         self.ib_client = None
         self.ec2_client = None
 
+        # maybe add the GIT_CLONE_OPTS parameter
+        if 'GIT_CLONE_OPTS' in os.environ:
+            self.parameters.append({'ParameterKey': 'GitCloneOpts', 'ParameterValue': os.environ['GIT_CLONE_OPTS']})
+
     def replace_stack(self) -> None:
         """
         Replace the stack (i.e., delete and create)