diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 09c171110..7be2b25cf 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -139,6 +139,7 @@ jobs: - if: ${{needs.get.outputs.filelist}} name: Check markdown meta. run: | + pip3 install titlecase shopt -s globstar extglob python3 checks/run_meta_check.py ${{needs.get.outputs.filelist}} slurmcheck: diff --git a/checks/run_meta_check.py b/checks/run_meta_check.py index 20ac10cee..af83556ce 100755 --- a/checks/run_meta_check.py +++ b/checks/run_meta_check.py @@ -11,6 +11,7 @@ import yaml import os import time +from titlecase import titlecase from pathlib import Path # Ignore files if they match this regex @@ -27,7 +28,7 @@ MAX_TITLE_LENGTH = 28 # As font isn't monospace, this is only approx MAX_HEADER_LENGTH = 32 # minus 2 per extra header level -MIN_TAGS = 2 +MIN_TAGS = 1 RANGE_SIBLING = [4, 8] DOC_ROOT = "docs" @@ -324,6 +325,14 @@ def title_length(): Try to keep it under {MAX_TITLE_LENGTH} characters to avoid word wrapping in the nav.", } +def title_capitalisation(): + correct_title = titlecase(title) + if title != correct_title: + yield { + "line": _get_lineno(r"^title:.*$"), + "message": f"Title '{title}' uses incorrect capitalisation. \ +'{correct_title}' is preferred", + } def minimum_tags(): if "tags" not in meta or not isinstance(meta["tags"], list): @@ -407,6 +416,7 @@ def dynamic_slurm_link(): ENDCHECKS = [ title_redundant, title_length, + title_capitalisation, meta_missing_description, meta_unexpected_key, minimum_tags, diff --git a/docs/.pages.yml b/docs/.pages.yml index fafded3d7..9ab168cbe 100644 --- a/docs/.pages.yml +++ b/docs/.pages.yml @@ -1,6 +1,8 @@ nav: + - Announcements - Getting_Started - - General - - Scientific_Computing + - Software + - Batch_Computing + - Interactive_Computing - Storage - Service_Subscriptions diff --git a/docs/General/Announcements/.pages.yml b/docs/Announcements/.pages.yml similarity index 79% rename from docs/General/Announcements/.pages.yml rename to docs/Announcements/.pages.yml index ee5ce4812..4ad570eab 100644 --- a/docs/General/Announcements/.pages.yml +++ b/docs/Announcements/.pages.yml @@ -3,4 +3,6 @@ nav: - Autodeletion_returning_for_scratch_filesystem.md - December_holiday_support_restrictions.md - Identity_Changes_for_Crown_Research_Institutes.md + - Known_Issues_HPC3.md + - Release_Notes diff --git a/docs/General/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md b/docs/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md similarity index 96% rename from docs/General/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md rename to docs/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md index dfdd321b3..00ebabca5 100644 --- a/docs/General/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md +++ b/docs/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md @@ -19,7 +19,7 @@ sources of self-service support: - Changes to system status are reported via our [System Status page](https://status.nesi.org.nz/ "https://status.nesi.org.nz/"). You can also subscribe for notifications of system updates and - unplanned outages sent straight to your inbox. [Sign up here.](../../Getting_Started/Getting_Help/System_status.md) + unplanned outages sent straight to your inbox. [Sign up here.](../Getting_Started/Getting_Help/System_status.md) - [Consult our User Documentation](https://www.docs.nesi.org.nz) pages for instructions and guidelines for using the systems. diff --git a/docs/General/Announcements/Autodeletion_returning_for_scratch_filesystem.md b/docs/Announcements/Autodeletion_returning_for_scratch_filesystem.md similarity index 100% rename from docs/General/Announcements/Autodeletion_returning_for_scratch_filesystem.md rename to docs/Announcements/Autodeletion_returning_for_scratch_filesystem.md diff --git a/docs/General/Announcements/December_holiday_support_restrictions.md b/docs/Announcements/December_holiday_support_restrictions.md similarity index 70% rename from docs/General/Announcements/December_holiday_support_restrictions.md rename to docs/Announcements/December_holiday_support_restrictions.md index 2aa9ebb9d..fe4d95f26 100644 --- a/docs/General/Announcements/December_holiday_support_restrictions.md +++ b/docs/Announcements/December_holiday_support_restrictions.md @@ -29,8 +29,8 @@ END: 9:00am 5 January 2026 Self-service support
Available 24/7 - Email: support@nesi.org.nz - - Sign up for system status updates for advance warning of any system updates or unplanned outages.
- Consult our User Documentation pages for instructions and guidelines for using the systems.
- Visit our YouTube channel  for introductory training webinars. + Email: support@nesi.org.nz + - Sign up for system status updates for advance warning of any system updates or unplanned outages.
- Consult our User Documentation pages for instructions and guidelines for using the systems.
- Visit our YouTube channel  for introductory training webinars. @@ -45,7 +45,7 @@ END: 9:00am 5 January 2026 Self-service support
Available 24/7 - Phone: 0508 466 466
Email: help@reannz.co.nz + Phone: 0508 466 466
Email: help@reannz.co.nz - Sign up for system status updates for advance warning of any system updates or unplanned outages.
- Check the REANNZ weathermap for a real-time view of network operations. diff --git a/docs/General/Announcements/Identity_Changes_for_Crown_Research_Institutes.md b/docs/Announcements/Identity_Changes_for_Crown_Research_Institutes.md similarity index 100% rename from docs/General/Announcements/Identity_Changes_for_Crown_Research_Institutes.md rename to docs/Announcements/Identity_Changes_for_Crown_Research_Institutes.md diff --git a/docs/General/Announcements/Known_Issues_HPC3.md b/docs/Announcements/Known_Issues_HPC3.md similarity index 95% rename from docs/General/Announcements/Known_Issues_HPC3.md rename to docs/Announcements/Known_Issues_HPC3.md index cabd1e253..b65d58070 100644 --- a/docs/General/Announcements/Known_Issues_HPC3.md +++ b/docs/Announcements/Known_Issues_HPC3.md @@ -8,7 +8,7 @@ tags: Below is a list issues that we're actively working on. We hope to have these resolved soon. This is intended to be a temporary page. -For differences between the new platforms and Mahuika, see the more permanent [differences from Mahuika](../../General/FAQs/Mahuika_HPC3_Differences.md). +For differences between the new platforms and Mahuika, see the more permanent [differences from Mahuika](../Getting_Started/FAQs/Mahuika_HPC3_Differences.md). !!! info "Recently fixed" diff --git a/docs/General/Release_Notes/index.md b/docs/Announcements/Release_Notes/index.md similarity index 86% rename from docs/General/Release_Notes/index.md rename to docs/Announcements/Release_Notes/index.md index d0908b597..b6c97fc85 100644 --- a/docs/General/Release_Notes/index.md +++ b/docs/Announcements/Release_Notes/index.md @@ -1,11 +1,7 @@ --- created_at: '2021-02-23T19:52:34Z' tags: [] -vote_count: 0 -vote_sum: 0 title: Release Notes -zendesk_article_id: 360003507115 -zendesk_section_id: 360000437436 --- NeSI publishes release notes for applications, 3rd party applications @@ -23,7 +19,7 @@ be located under Storage, Long-Term Storage ## 3rd party applications -3rd party applications listed under [Supported Applications](../../Scientific_Computing/Supported_Applications/index.md) +3rd party applications listed under [Supported Applications](../../Software/Available_Applications/index.md) have child pages with details about the available versions on NeSI, and a reference to the vendor release notes or documentation. diff --git a/docs/Scientific_Computing/Batch_Jobs/.pages.yml b/docs/Batch_Computing/.pages.yml similarity index 82% rename from docs/Scientific_Computing/Batch_Jobs/.pages.yml rename to docs/Batch_Computing/.pages.yml index d3a7c0800..1490d5275 100644 --- a/docs/Scientific_Computing/Batch_Jobs/.pages.yml +++ b/docs/Batch_Computing/.pages.yml @@ -1,10 +1,10 @@ nav: + - Submitting_your_first_job.md - Hardware.md - Job_prioritisation.md - SLURM-Best_Practice.md - Using_GPUs.md - Job_Checkpointing.md - - Slurm_Interactive_Sessions.md - Fair_Share.md - Checksums.md - "*" diff --git a/docs/Scientific_Computing/Batch_Jobs/Checking_resource_usage.md b/docs/Batch_Computing/Checking_resource_usage.md similarity index 100% rename from docs/Scientific_Computing/Batch_Jobs/Checking_resource_usage.md rename to docs/Batch_Computing/Checking_resource_usage.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Checksums.md b/docs/Batch_Computing/Checksums.md similarity index 100% rename from docs/Scientific_Computing/Batch_Jobs/Checksums.md rename to docs/Batch_Computing/Checksums.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Fair_Share.md b/docs/Batch_Computing/Fair_Share.md similarity index 98% rename from docs/Scientific_Computing/Batch_Jobs/Fair_Share.md rename to docs/Batch_Computing/Fair_Share.md index 9d22c77e0..39c04f38b 100644 --- a/docs/Scientific_Computing/Batch_Jobs/Fair_Share.md +++ b/docs/Batch_Computing/Fair_Share.md @@ -18,7 +18,7 @@ Your *Fair Share score* is a number between **0** and **1**. Projects with a **larger** Fair Share score receive a **higher priority** in the queue. -A project is given an [allocation of compute units](../../Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md) +A project is given an [allocation of compute units](../Getting_Started/Allocations/What_is_an_allocation.md) over a given **period**. An institution also has a percentage **Fair Share entitlement** of each machine's deliverable capacity over that same period. diff --git a/docs/Scientific_Computing/Batch_Jobs/Hardware.md b/docs/Batch_Computing/Hardware.md similarity index 100% rename from docs/Scientific_Computing/Batch_Jobs/Hardware.md rename to docs/Batch_Computing/Hardware.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Job_Checkpointing.md b/docs/Batch_Computing/Job_Checkpointing.md similarity index 100% rename from docs/Scientific_Computing/Batch_Jobs/Job_Checkpointing.md rename to docs/Batch_Computing/Job_Checkpointing.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Job_Limits.md b/docs/Batch_Computing/Job_Limits.md similarity index 100% rename from docs/Scientific_Computing/Batch_Jobs/Job_Limits.md rename to docs/Batch_Computing/Job_Limits.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Job_prioritisation.md b/docs/Batch_Computing/Job_prioritisation.md similarity index 98% rename from docs/Scientific_Computing/Batch_Jobs/Job_prioritisation.md rename to docs/Batch_Computing/Job_prioritisation.md index cac7da8c8..61c78149c 100644 --- a/docs/Scientific_Computing/Batch_Jobs/Job_prioritisation.md +++ b/docs/Batch_Computing/Job_prioritisation.md @@ -29,7 +29,7 @@ jobs, but is limited to one small job per user at a time: no more than Job priority decreases whenever the project uses more core-hours than expected, across all partitions. -This [Fair Share](../../Scientific_Computing/Batch_Jobs/Fair_Share.md) +This [Fair Share](Fair_Share.md) policy means that projects that have consumed many CPU core hours in the recent past compared to their expected rate of use (either by submitting and running many jobs, or by submitting and running large jobs) will diff --git a/docs/Scientific_Computing/Batch_Jobs/SLURM-Best_Practice.md b/docs/Batch_Computing/SLURM-Best_Practice.md similarity index 92% rename from docs/Scientific_Computing/Batch_Jobs/SLURM-Best_Practice.md rename to docs/Batch_Computing/SLURM-Best_Practice.md index c7cae8414..c0dae4dab 100644 --- a/docs/Scientific_Computing/Batch_Jobs/SLURM-Best_Practice.md +++ b/docs/Batch_Computing/SLURM-Best_Practice.md @@ -44,7 +44,7 @@ etc). ### Memory (RAM) If you request more memory (RAM) than you need for your job, it -[will wait longer in the queue and will be more expensive when it runs](../../General/FAQs/Why_is_my_job_taking_a_long_time_to_start.md). +[will wait longer in the queue and will be more expensive when it runs](../Getting_Started/FAQs/Why_is_my_job_taking_a_long_time_to_start.md). On the other hand, if you don't request enough memory, the job may be killed for attempting to exceed its allocated memory limits. @@ -53,7 +53,7 @@ your program will need at peak memory usage. We also recommend using `--mem` instead of `--mem-per-cpu` in most cases. There are a few kinds of jobs for which `--mem-per-cpu` is more -suitable. See [our article on how to request memory](../../General/FAQs/How_do_I_request_memory.md) +suitable. See [our article on how to request memory](../Getting_Started/FAQs/How_do_I_request_memory.md) for more information. ## Parallelism @@ -77,4 +77,4 @@ job array in a single command) A low fairshare score will affect your jobs priority in the queue, learn more about how to effectively use your allocation, -[Fair Share](../../Scientific_Computing/Batch_Jobs/Fair_Share.md). +[Fair Share](Fair_Share.md). diff --git a/docs/Getting_Started/Next_Steps/Submitting_your_first_job.md b/docs/Batch_Computing/Submitting_your_first_job.md similarity index 100% rename from docs/Getting_Started/Next_Steps/Submitting_your_first_job.md rename to docs/Batch_Computing/Submitting_your_first_job.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Temporary_directories.md b/docs/Batch_Computing/Temporary_directories.md similarity index 100% rename from docs/Scientific_Computing/HPC_Software_Environment/Temporary_directories.md rename to docs/Batch_Computing/Temporary_directories.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Using_GPUs.md b/docs/Batch_Computing/Using_GPUs.md similarity index 91% rename from docs/Scientific_Computing/Batch_Jobs/Using_GPUs.md rename to docs/Batch_Computing/Using_GPUs.md index 878ac420b..0d78cc67d 100644 --- a/docs/Scientific_Computing/Batch_Jobs/Using_GPUs.md +++ b/docs/Batch_Computing/Using_GPUs.md @@ -18,7 +18,7 @@ This page provides generic information about how to access GPUs through the Slur ## Request GPU resources using Slurm -To request a GPU for your [Slurm job](../../Getting_Started/Next_Steps/Submitting_your_first_job.md), add +To request a GPU for your [Slurm job](Submitting_your_first_job.md), add the following option in the header of your submission script: ```sl @@ -28,7 +28,7 @@ the following option in the header of your submission script: You can specify the type and number of GPU you need using the following syntax -``` sl +```sl #SBATCH --gpus-per-node=: ``` @@ -81,7 +81,7 @@ It is recommended to specify the exact GPU type required; otherwise, the job may You can also use the `--gpus-per-node`option in -[Slurm interactive sessions](./Slurm_Interactive_Sessions.md), +[Slurm interactive sessions](../Interactive_Computing/Slurm_Interactive_Sessions.md), with the `srun` and `salloc` commands. For example: ``` sh @@ -111,7 +111,7 @@ duration of 30 minutes. ## Load CUDA and cuDNN modules To use an Nvidia GPU card with your application, you need to load the -driver and the CUDA toolkit via the [environment modules](../Supported_Applications/index.md) +driver and the CUDA toolkit via the [environment modules](../Software/Available_Applications/index.md) mechanism: ``` sh @@ -229,12 +229,12 @@ CUDA_VISIBLE_DEVICES=0 The following pages provide additional information for supported applications: -- [ABAQUS](../Supported_Applications/ABAQUS.md#examples) -- [GROMACS](../Supported_Applications/GROMACS.md) -- [Lambda Stack](../Supported_Applications/Lambda_Stack.md) -- [Matlab](../Supported_Applications/MATLAB.md#using-gpus) -- [TensorFlow on GPUs](../Supported_Applications/TensorFlow_on_GPUs.md) +- [ABAQUS](../Software/Available_Applications/ABAQUS.md#examples) +- [GROMACS](../Software/Available_Applications/GROMACS.md) +- [Lambda Stack](../Software/Available_Applications/Lambda_Stack.md) +- [Matlab](../Software/Available_Applications/MATLAB.md#using-gpus) +- [TensorFlow on GPUs](../Software/Available_Applications/TensorFlow_on_GPUs.md) And programming toolkits: -- [NVIDIA GPU Containers](../HPC_Software_Environment/NVIDIA_GPU_Containers.md) +- [NVIDIA GPU Containers](../Software/Containers/NVIDIA_GPU_Containers.md) diff --git a/docs/General/.pages.yml b/docs/General/.pages.yml deleted file mode 100644 index 220bf4fff..000000000 --- a/docs/General/.pages.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -nav: -- Announcements -- FAQs -- Policy -- Release_Notes diff --git a/docs/Getting_Started/.pages.yml b/docs/Getting_Started/.pages.yml index 799298a11..f8792971b 100644 --- a/docs/Getting_Started/.pages.yml +++ b/docs/Getting_Started/.pages.yml @@ -1,8 +1,9 @@ --- nav: - - Accounts, Projects and Allocations : Accounts-Projects_and_Allocations + - Creating_an_Account.md + - Projects + - Allocations - Accessing_the_HPCs - - Next_Steps - Getting_Help - Cheat_Sheets - "*" diff --git a/docs/Getting_Started/Accessing_the_HPCs/Connecting_to_the_Cluster.md b/docs/Getting_Started/Accessing_the_HPCs/Connecting_to_the_Cluster.md index 33ada6fbf..f373b7fee 100644 --- a/docs/Getting_Started/Accessing_the_HPCs/Connecting_to_the_Cluster.md +++ b/docs/Getting_Started/Accessing_the_HPCs/Connecting_to_the_Cluster.md @@ -8,7 +8,7 @@ tags: --- !!! prerequisite - - Have an [active account and project](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md). + - Have an [active account and project](../Creating_an_Account.md). Before you can start submitting work you will need some way of connecting to the NeSI clusters. @@ -27,7 +27,7 @@ operating system and level of experience. !!! tip "What next?" - More info on - [NeSI OnDemand](../../Scientific_Computing/Interactive_computing_with_OnDemand/how_to_guide.md) + [NeSI OnDemand](../../Interactive_Computing/OnDemand/how_to_guide.md) - Visit [ondemand.nesi.org.nz](https://ondemand.nesi.org.nz/). ## Linux or Mac OS @@ -40,12 +40,12 @@ installed, usually called, "Terminal." To find it, simply search for Congratulations! You are ready to move to the next step. !!! prerequisite "What next?" - Setting up your [Default Terminal](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) + Setting up your [Default Terminal](Standard_Terminal_Setup.md) ### VSCode The inbuilt 'remotes' plugin allows connecting to remote hosts. -If you have set up your `~/.ssh/config` as described in [Standard_Terminal_Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md), +If you have set up your `~/.ssh/config` as described in [Standard_Terminal_Setup](Standard_Terminal_Setup.md), VSCode will detect this and show configured hosts in the 'Remote Explorer' Tab. ## Windows @@ -69,8 +69,8 @@ different options, listed in order of preference. !!! tip "What next?" - Enabling - [WSL](../../Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md) - - Setting up the [Ubuntu Terminal](../../Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md) + [WSL](Windows_Subsystem_for_Linux_WSL.md) + - Setting up the [Ubuntu Terminal](Windows_Subsystem_for_Linux_WSL.md) ### VSCode @@ -91,7 +91,7 @@ VSCode can be used with WSL or without. institution's IT team supports MobaXTerm. !!! tip "What next?" - Setting up - [MobaXterm](../../Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md) + [MobaXterm](MobaXterm_Setup_Windows.md) ### Using a Virtual Machine @@ -123,7 +123,7 @@ for new users. !!! tip "What next?" - Setting up - [WinSCP](../../Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md) + [WinSCP](WinSCP-PuTTY_Setup_Windows.md) ### Git Bash @@ -141,7 +141,7 @@ primary terminal. All Windows computers have PowerShell installed, however it will only be useful to you if Windows Subsystem for Linux (WSL) is also enabled, instructions can be found at -[Windows_Subsystem_for_Linux_WSL](../../Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md). +[Windows_Subsystem_for_Linux_WSL](Windows_Subsystem_for_Linux_WSL.md). Like Git Bash, PowerShell is perfectly adequate for testing your login or setting up your password, but lacks many of the features of diff --git a/docs/Getting_Started/Accessing_the_HPCs/First_Time_Login.md b/docs/Getting_Started/Accessing_the_HPCs/First_Time_Login.md index 9ef529346..2d2be4f14 100644 --- a/docs/Getting_Started/Accessing_the_HPCs/First_Time_Login.md +++ b/docs/Getting_Started/Accessing_the_HPCs/First_Time_Login.md @@ -11,8 +11,8 @@ tags: --- !!! prerequisite - - Have an [account](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md). - - Be a member of an [active project](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md). + - Have an [account](../Creating_an_Account.md). + - Be a member of an [active project](../Creating_an_Account.md). - Have a device with an authentication app. !!! note @@ -20,7 +20,7 @@ tags: NeSI’s services and technologies are now hosted by REANNZ as a national eResearch Infrastructure Platform. Some of our tools (as pictured in the screenshot below) will retain a ‘NeSI’ brand as we transition our services and develop a longer-term strategy for this integrated platform. -1. Log into [my.nesi](my.nesi.org.nz) +1. Log into [my.nesi](http://my.nesi.org.nz) 2. Go to [**OnDemand**](https://ondemand.nesi.org.nz/). It will automatically take you to the Tuakiri login screen. ![alt text](../../assets/images/ondemand_login_0.png) @@ -28,7 +28,7 @@ tags: 3. Select your affiliated institution, and log in using your institutional account. Example below shows the University of Auckland login screen. ![alt text](../../assets/images/ondemand_login_1.png) -4. If you haven't logged into OnDemand or our HPC platforms before, you wil need to set up new authentication credentials. This is in addition to your institutional MFA process. +4. If you haven't logged into OnDemand or our HPC platforms before, you will need to set up new authentication credentials. This is in addition to your institutional MFA process. ![alt text](../../assets/images/ondemand_login_2.png) !!! note diff --git a/docs/Scientific_Computing/Terminal_Setup/Git_Bash_Windows.md b/docs/Getting_Started/Accessing_the_HPCs/Git_Bash_Windows.md similarity index 74% rename from docs/Scientific_Computing/Terminal_Setup/Git_Bash_Windows.md rename to docs/Getting_Started/Accessing_the_HPCs/Git_Bash_Windows.md index 7ae5e9c46..167320218 100644 --- a/docs/Scientific_Computing/Terminal_Setup/Git_Bash_Windows.md +++ b/docs/Getting_Started/Accessing_the_HPCs/Git_Bash_Windows.md @@ -9,8 +9,8 @@ title: Git Bash (Windows) --- !!! prerequisite - - Have a [NeSI account.](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md). - - Be a member of an [active project.](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md) + - Have a [NeSI account.](../Creating_an_Account.md). + - Be a member of an [active project.](../Projects/Applying_to_Join_a_Project.md) ## First time setup @@ -40,7 +40,7 @@ credentials every time you open a new terminal or try to move a file.* scp nesi:~/ ``` -For more info visit [data transfer](../../Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md). +For more info visit [data transfer](../../Storage/Moving_files_to_and_from_the_cluster.md). !!! prerequisite "What Next?" - [Standard Terminal Setup](Standard_Terminal_Setup.md) diff --git a/docs/Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md b/docs/Getting_Started/Accessing_the_HPCs/MobaXterm_Setup_Windows.md similarity index 92% rename from docs/Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md rename to docs/Getting_Started/Accessing_the_HPCs/MobaXterm_Setup_Windows.md index 6c4d686a8..5d0485597 100644 --- a/docs/Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md +++ b/docs/Getting_Started/Accessing_the_HPCs/MobaXterm_Setup_Windows.md @@ -12,7 +12,7 @@ description: How to set up cluster access using MobaXterm It is recommended to use [OnDemand](https://ondemand.nesi.org.nz/) for file browsing, up and downloading and terminal access if you would normally have used MobaXterm. !!! prerequisite - - Have an [active account and project.](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) + - Have an [active account and project.](../Creating_an_Account.md) - [Download MobaXterm](https://mobaxterm.mobatek.net/download-home-edition.html) - Followed the steps in [Standard Terminal](Standard_Terminal_Setup.md). @@ -22,10 +22,10 @@ description: How to set up cluster access using MobaXterm - Otherwise, choose freely the Portable or Installer Edition. !!! prerequisite "What Next?" - - [Moving files to/from a cluster.](../../Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md) + - [Moving files to/from a cluster.](../../Storage/Moving_files_to_and_from_the_cluster.md) The interactive login configuration for MobaXterm is not compatable with the current web-based authentication method. If you wish to use MobaXterm as your SSH client you therefore need to use a non-interactive setup. -This can be done by following a modified version of the instructions for setting up the [the standard terminal setup described on this support page](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md). +This can be done by following a modified version of the instructions for setting up the [the standard terminal setup described on this support page](Standard_Terminal_Setup.md). ## First time setup diff --git a/docs/Getting_Started/Accessing_the_HPCs/Port_Forwarding.md b/docs/Getting_Started/Accessing_the_HPCs/Port_Forwarding.md index 0b2015271..1a73ef3e2 100644 --- a/docs/Getting_Started/Accessing_the_HPCs/Port_Forwarding.md +++ b/docs/Getting_Started/Accessing_the_HPCs/Port_Forwarding.md @@ -6,7 +6,7 @@ tags: --- !!! prerequisite - Have your [connection to the NeSI cluster](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) configured + Have your [connection to the NeSI cluster](Standard_Terminal_Setup.md) configured Some applications only accept connections from internal ports (i.e a port on the same local network), if you are running one such application @@ -24,12 +24,12 @@ to `127.0.0.1`. The alias `localhost` can also be used in most cases. **Host Alias:** An alias for the socket of your main connection to the cluster, `nesi` if you have set up your ssh config file as -described in [Standard Terminal Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md). +described in [Standard Terminal Setup](Standard_Terminal_Setup.md). **Remote Port:** The port number you will use on the remote machine (in this case the NeSI cluster) !!! note - The following examples use aliases as set up in [standard terminal setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md). + The following examples use aliases as set up in [standard terminal setup](Standard_Terminal_Setup.md). This allows the forwarding from your local machine to the NeSI cluster, without having to re-tunnel through the lander node. @@ -205,4 +205,4 @@ ssh -Nf -R 6676:localhost:6676 ${SLURM_SUBMIT_HOST} ``` !!! tip "What Next?" - - [Paraview](../../Scientific_Computing/Supported_Applications/ParaView.md) + - [Paraview](../../Software/Available_Applications/ParaView.md) diff --git a/docs/Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md b/docs/Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md similarity index 96% rename from docs/Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md rename to docs/Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md index 2458f05d7..bc6ee448e 100644 --- a/docs/Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md +++ b/docs/Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md @@ -7,7 +7,7 @@ description: How to setup your ssh config file in order to connect to the HPC cl --- !!! prerequisite - - Have an [active account and project.](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) + - Have an [active account and project.](../Creating_an_Account.md) - Have one of: - Built in Linux/Mac terminal - [Windows Subsystem for Linux](Windows_Subsystem_for_Linux_WSL.md) @@ -184,5 +184,5 @@ You should now be able to login with only a single authentication prompt. [Watch a demo](https://www.youtube.com/embed/IKihbN-QlIA?si=N93PPPsi85jPYV7k). !!! prerequisite "What Next?" - - [Moving files to/from a cluster.](../../Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md) + - [Moving files to/from a cluster.](../../Storage/Moving_files_to_and_from_the_cluster.md) - Setting up an [X-Server](./X11.md) (optional). diff --git a/docs/Scientific_Computing/Terminal_Setup/VSCode.md b/docs/Getting_Started/Accessing_the_HPCs/VSCode.md similarity index 95% rename from docs/Scientific_Computing/Terminal_Setup/VSCode.md rename to docs/Getting_Started/Accessing_the_HPCs/VSCode.md index 685e75a7a..b35fc7729 100644 --- a/docs/Scientific_Computing/Terminal_Setup/VSCode.md +++ b/docs/Getting_Started/Accessing_the_HPCs/VSCode.md @@ -83,7 +83,7 @@ Clicking on these will open a connection to that machine, you will then be promp You may find that VSCode is not utilising your preferred versions of software (e.g. when debugging or linting your Python code). -As the NeSI cluster utilises [Environment Modules](../../Getting_Started/Next_Steps/Submitting_your_first_job.md#environment-modules), changing the executable used is not just a matter of changing the path in VSCode configuration, as the libraries required will not be loaded. +As the NeSI cluster utilises [Environment Modules](../../Batch_Computing/Submitting_your_first_job.md#environment-modules), changing the executable used is not just a matter of changing the path in VSCode configuration, as the libraries required will not be loaded. The only way to make sure that VSCode has access to a suitable environment, is to load the required modules in your `~/.bashrc` diff --git a/docs/Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md b/docs/Getting_Started/Accessing_the_HPCs/WinSCP-PuTTY_Setup_Windows.md similarity index 94% rename from docs/Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md rename to docs/Getting_Started/Accessing_the_HPCs/WinSCP-PuTTY_Setup_Windows.md index 74ca5dcef..f09671ecb 100644 --- a/docs/Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md +++ b/docs/Getting_Started/Accessing_the_HPCs/WinSCP-PuTTY_Setup_Windows.md @@ -8,7 +8,7 @@ title: WinSCP/PuTTY Setup (Windows) --- !!! prerequisite - - Have an [active account and project.](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) + - Have an [active account and project.](../Creating_an_Account.md) - Be using the Windows operating system. WinSCP is an SCP client for windows implementing the SSH protocol from @@ -129,5 +129,5 @@ for a single transfer'. with login authentication. !!! prerequisite "What Next?" - - [Moving files to and from the cluster](../../Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md) + - [Moving files to and from the cluster](../../Storage/Moving_files_to_and_from_the_cluster.md) - [X11 on NeSI](./X11.md)(optional). diff --git a/docs/Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md b/docs/Getting_Started/Accessing_the_HPCs/Windows_Subsystem_for_Linux_WSL.md similarity index 96% rename from docs/Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md rename to docs/Getting_Started/Accessing_the_HPCs/Windows_Subsystem_for_Linux_WSL.md index 140d7356b..de65c1f6b 100644 --- a/docs/Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md +++ b/docs/Getting_Started/Accessing_the_HPCs/Windows_Subsystem_for_Linux_WSL.md @@ -85,4 +85,4 @@ ln -s /mnt/c/Users/YourWindowsUsername/ WinFS ``` !!! prerequisite What "Next?" - - Set up your [SSH config file](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md). + - Set up your [SSH config file](Standard_Terminal_Setup.md). diff --git a/docs/Scientific_Computing/Terminal_Setup/X11.md b/docs/Getting_Started/Accessing_the_HPCs/X11.md similarity index 100% rename from docs/Scientific_Computing/Terminal_Setup/X11.md rename to docs/Getting_Started/Accessing_the_HPCs/X11.md diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/.pages.yml b/docs/Getting_Started/Accounts-Projects_and_Allocations/.pages.yml deleted file mode 100644 index f9c63598b..000000000 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/.pages.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -nav: - - Creating_an_Account_Profile.md - - Applying_for_a_new_project.md - - Applying_to_join_a_project.md - - What_is_an_allocation.md - - Quarterly_allocation_periods.md - - "*" diff --git a/docs/Getting_Started/Allocations/.pages.yml b/docs/Getting_Started/Allocations/.pages.yml new file mode 100644 index 000000000..d21d35fba --- /dev/null +++ b/docs/Getting_Started/Allocations/.pages.yml @@ -0,0 +1,5 @@ +--- +nav: + - What_is_an_allocation.md + - Allocations_and_Extensions.md + - "*" diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Allocations_and_Extensions.md b/docs/Getting_Started/Allocations/Allocations_and_Extensions.md similarity index 96% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Allocations_and_Extensions.md rename to docs/Getting_Started/Allocations/Allocations_and_Extensions.md index 3d02afa79..2408dda59 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Allocations_and_Extensions.md +++ b/docs/Getting_Started/Allocations/Allocations_and_Extensions.md @@ -1,11 +1,9 @@ --- created_at: '2018-05-18T02:34:03Z' -tags: [] -title: Project Extensions and New Allocations on Existing Projects -vote_count: 1 -vote_sum: 1 -zendesk_article_id: 360000202196 -zendesk_section_id: 360000196195 +tags: +- projects +- allocations +title: Allocations & Extensions --- NeSI recognises that research programmes often continue over several diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Quarterly_allocation_periods.md b/docs/Getting_Started/Allocations/Quarterly_allocation_periods.md similarity index 93% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Quarterly_allocation_periods.md rename to docs/Getting_Started/Allocations/Quarterly_allocation_periods.md index d24eb366b..c744f4316 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Quarterly_allocation_periods.md +++ b/docs/Getting_Started/Allocations/Quarterly_allocation_periods.md @@ -1,11 +1,8 @@ --- created_at: '2021-09-14T03:20:56Z' -tags: [] -title: Quarterly allocation periods -vote_count: 0 -vote_sum: 0 -zendesk_article_id: 4406437522703 -zendesk_section_id: 360000196195 +tags: +- allocations +title: Quarterly Allocation Periods --- Applications for new allocations on existing projects are accepted and @@ -54,4 +51,4 @@ month. wait for the following call before your request is considered. If you have questions about the review cycles or other steps involved -with getting access to NeSI, {% include "partials/support_request.html" %} \ No newline at end of file +with getting access to NeSI, {% include "partials/support_request.html" %} diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md b/docs/Getting_Started/Allocations/What_is_an_allocation.md similarity index 93% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md rename to docs/Getting_Started/Allocations/What_is_an_allocation.md index ed867cde6..3bd8535b1 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md +++ b/docs/Getting_Started/Allocations/What_is_an_allocation.md @@ -1,10 +1,9 @@ --- created_at: '2020-02-25T02:35:13Z' tags: - - Allocation - Allocations - Compute -title: What is an allocation? +title: What is an Allocation? --- Because NeSI's resources are limited, we manage access to our resources @@ -15,15 +14,15 @@ different allocation criteria. An allocation will come from one of our allocation classes. We will decide what class of allocation is most suitable for you and your -research programme, however you're welcome to review [our article on -allocation classes](../../General/Policy/Allocation_classes.md) +research programme, however you're welcome to review +[our article on allocation classes](../Policy/Allocation_classes.md) to find out what class you're likely eligible for. ## An important note on CPU hour allocations You may continue to submit jobs even if you have used all your CPU-hour allocation. The effect of 0 remaining CPU hours allocation is a -[lower fairshare](../../Scientific_Computing/Batch_Jobs/Fair_Share.md), +[lower fairshare](../../Batch_Computing/Fair_Share.md), not the inability to use CPUs. Your ability to submit jobs will only be removed when your project's allocation expires, not when core-hours are exhausted. @@ -38,7 +37,7 @@ plus one kind of compute allocation) in order to be valid and active. Compute allocations are expressed in terms of a number of units, to be consumed or reserved between a set start date and time and a set end date and time. For allocations of computing power, we use [Fair -Share](../../Scientific_Computing/Batch_Jobs/Fair_Share.md) +Share](../../Batch_Computing/Fair_Share.md) to balance work between different projects. NeSI allocations and the relative "prices" of resources used by those allocations should not be taken as any indicator of the real NZD costs of purchasing or running diff --git a/docs/Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md b/docs/Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md index d14f25249..56a6bb0ff 100644 --- a/docs/Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md +++ b/docs/Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md @@ -8,7 +8,7 @@ description: Quick list of the most commonly used Slurm commands, flags, and env --- If you are unsure about using our job scheduler Slurm, more details can -be found on [Submitting_your_first_job](../../Getting_Started/Next_Steps/Submitting_your_first_job.md). +be found on [Submitting_your_first_job](../../Batch_Computing/Submitting_your_first_job.md). ## Slurm Commands @@ -58,10 +58,10 @@ an '=' sign e.g. `#SBATCH --account=nesi99999` or a space e.g. | | | | | --------------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | | `--nodes` | ``#SBATCH --nodes=2`` | Will request tasks be run across 2 nodes. | -| `--ntasks` | ``#SBATCH --ntasks=2 `` | Will start 2 [MPI](../../Getting_Started/Next_Steps/Parallel_Execution.md) tasks. | +| `--ntasks` | ``#SBATCH --ntasks=2 `` | Will start 2 [MPI](../../Software/Parallel_Computing/Parallel_Execution.md) tasks. | | `--ntasks-per-node` | `#SBATCH --ntasks-per-node=1` | Will start 1 task per requested node. | -| `--cpus-per-task` | `#SBATCH --cpus-per-task=10` | Will request 10 [*logical* CPUs](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md) per task. | -| `--mem-per-cpu` | `#SBATCH --mem-per-cpu=512MB` | Memory Per *logical* CPU. `--mem` Should be used if shared memory job. See [How do I request memory?](../../General/FAQs/How_do_I_request_memory.md) | +| `--cpus-per-task` | `#SBATCH --cpus-per-task=10` | Will request 10 [*logical* CPUs](../../Software/Parallel_Computing/Hyperthreading.md) per task. | +| `--mem-per-cpu` | `#SBATCH --mem-per-cpu=512MB` | Memory Per *logical* CPU. `--mem` Should be used if shared memory job. See [How do I request memory?](../FAQs/How_do_I_request_memory.md) | | --array | `#SBATCH --array=1-5` | Will submit job 5 times each with a different `$SLURM_ARRAY_TASK_ID` (1,2,3,4,5). | | | `#SBATCH --array=0-20:5` | Will submit job 5 times each with a different `$SLURM_ARRAY_TASK_ID` (0,5,10,15,20). | | | `#SBATCH --array=1-100%10` | Will submit 1 though to 100 jobs but no more than 10 at once. | @@ -71,9 +71,9 @@ an '=' sign e.g. `#SBATCH --account=nesi99999` or a space e.g. | | | | | -- | -- | -- | | `--qos` | `#SBATCH --qos=debug` | Adding this line gives your job a high priority. *Limited to one job at a time, max 15 minutes*. | -| `--profile` | `#SBATCH --profile=ALL` | Allows generation of a .h5 file containing job profile information. See [Slurm Native Profiling](../../Scientific_Computing/Profiling_and_Debugging/Slurm_Native_Profiling.md) | +| `--profile` | `#SBATCH --profile=ALL` | Allows generation of a .h5 file containing job profile information. See [Slurm Native Profiling](../../Software/Profiling_and_Debugging/Slurm_Native_Profiling.md) | | `--dependency` | `#SBATCH --dependency=afterok:123456789` | Will only start after the job 123456789 has completed. | -| `--hint` | `#SBATCH --hint=nomultithread` | Disables [hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md), be aware that this will significantly change how your job is defined. | +| `--hint` | `#SBATCH --hint=nomultithread` | Disables [hyperthreading](../../Software/Parallel_Computing/Hyperthreading.md), be aware that this will significantly change how your job is defined. | !!! tip Many options have a short (`-`) and long (`--`) form e.g. diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md b/docs/Getting_Started/Creating_an_Account.md similarity index 79% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md rename to docs/Getting_Started/Creating_an_Account.md index b6cdb54e1..0c38ff4e1 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md +++ b/docs/Getting_Started/Creating_an_Account.md @@ -10,7 +10,7 @@ tags: !!! prerequisite Either an active login at a Tuakiri member institution, or - [a Tuakiri Virtual Home account in respect of your current place of work or study](../../General/Policy/Account_Requests_for_non_Tuakiri_Members.md). + [a Tuakiri Virtual Home account in respect of your current place of work or study](./Policy/Account_Requests_for_non_Tuakiri_Members.md). 1. Access [my.nesi.org.nz](https://my.nesi.org.nz) via your browser and log in with either your institutional credentials, or your Tuakiri @@ -24,6 +24,6 @@ tags: our records. !!! prerequisite "What next?" - - [Apply for Access](./Applying_for_a_new_project.md), + - [Apply for Access](./Projects/Applying_for_a_New_Project.md), either submit an application for a new project or - [join an existing project](./Applying_to_join_a_project.md). + [join an existing project](./Projects/Applying_to_Join_a_Project.md). diff --git a/docs/General/FAQs/.pages.yml b/docs/Getting_Started/FAQs/.pages.yml similarity index 100% rename from docs/General/FAQs/.pages.yml rename to docs/Getting_Started/FAQs/.pages.yml diff --git a/docs/General/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md b/docs/Getting_Started/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md similarity index 90% rename from docs/General/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md rename to docs/Getting_Started/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md index d92bc0fbd..7503f49f2 100644 --- a/docs/General/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md +++ b/docs/Getting_Started/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md @@ -27,7 +27,7 @@ latter but not the former: test -r ~/.bashrc && . ~/.bashrc ``` -Please see the article, [.bashrc or.bash profile?](../../General/FAQs/What_are_my-bashrc_and-bash_profile_for.md) +Please see the article, [.bashrc or.bash profile?](What_are_my-bashrc_and-bash_profile_for.md) for more information. ## What about cron jobs? diff --git a/docs/General/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md b/docs/Getting_Started/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md similarity index 100% rename from docs/General/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md rename to docs/Getting_Started/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md diff --git a/docs/General/FAQs/Common_questions_about_the_platform_refresh.md b/docs/Getting_Started/FAQs/Common_questions_about_the_platform_refresh.md similarity index 100% rename from docs/General/FAQs/Common_questions_about_the_platform_refresh.md rename to docs/Getting_Started/FAQs/Common_questions_about_the_platform_refresh.md diff --git a/docs/General/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md b/docs/Getting_Started/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md similarity index 100% rename from docs/General/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md rename to docs/Getting_Started/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md diff --git a/docs/General/FAQs/How_busy_is_the_cluster.md b/docs/Getting_Started/FAQs/How_busy_is_the_cluster.md similarity index 100% rename from docs/General/FAQs/How_busy_is_the_cluster.md rename to docs/Getting_Started/FAQs/How_busy_is_the_cluster.md diff --git a/docs/General/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md b/docs/Getting_Started/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md similarity index 100% rename from docs/General/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md rename to docs/Getting_Started/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md diff --git a/docs/General/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md b/docs/Getting_Started/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md similarity index 100% rename from docs/General/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md rename to docs/Getting_Started/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md diff --git a/docs/General/FAQs/How_can_I_view_images_generated_on_the_cluster.md b/docs/Getting_Started/FAQs/How_can_I_view_images_generated_on_the_cluster.md similarity index 85% rename from docs/General/FAQs/How_can_I_view_images_generated_on_the_cluster.md rename to docs/Getting_Started/FAQs/How_can_I_view_images_generated_on_the_cluster.md index bc11ab1b8..a02a41041 100644 --- a/docs/General/FAQs/How_can_I_view_images_generated_on_the_cluster.md +++ b/docs/Getting_Started/FAQs/How_can_I_view_images_generated_on_the_cluster.md @@ -16,4 +16,4 @@ gm display myImage.png ``` This requires a [working X-11 -server](../../Scientific_Computing/Terminal_Setup/X11.md). +server](../Accessing_the_HPCs/X11.md). diff --git a/docs/General/FAQs/How_do_I_find_out_the_size_of_a_directory.md b/docs/Getting_Started/FAQs/How_do_I_find_out_the_size_of_a_directory.md similarity index 100% rename from docs/General/FAQs/How_do_I_find_out_the_size_of_a_directory.md rename to docs/Getting_Started/FAQs/How_do_I_find_out_the_size_of_a_directory.md diff --git a/docs/General/FAQs/How_do_I_fix_my_locale_and_language_settings.md b/docs/Getting_Started/FAQs/How_do_I_fix_my_locale_and_language_settings.md similarity index 100% rename from docs/General/FAQs/How_do_I_fix_my_locale_and_language_settings.md rename to docs/Getting_Started/FAQs/How_do_I_fix_my_locale_and_language_settings.md diff --git a/docs/General/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md b/docs/Getting_Started/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md similarity index 100% rename from docs/General/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md rename to docs/Getting_Started/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md diff --git a/docs/General/FAQs/How_do_I_request_memory.md b/docs/Getting_Started/FAQs/How_do_I_request_memory.md similarity index 97% rename from docs/General/FAQs/How_do_I_request_memory.md rename to docs/Getting_Started/FAQs/How_do_I_request_memory.md index 2446ba7cf..c79e944a1 100644 --- a/docs/General/FAQs/How_do_I_request_memory.md +++ b/docs/Getting_Started/FAQs/How_do_I_request_memory.md @@ -5,7 +5,7 @@ description: Instructions for requesting memory --- - `--mem`: Memory per node -- `--mem-per-cpu`: Memory per [logical CPU](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md) +- `--mem-per-cpu`: Memory per [logical CPU](../../Software/Parallel_Computing/Hyperthreading.md) In most circumstances, you should request memory using `--mem`. The exception is if you are running an MPI job that could be placed on more diff --git a/docs/General/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md b/docs/Getting_Started/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md similarity index 93% rename from docs/General/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md rename to docs/Getting_Started/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md index b5aa1af22..b69216bf5 100644 --- a/docs/General/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md +++ b/docs/Getting_Started/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md @@ -31,5 +31,5 @@ the file explorer in Jupyter from your downloads folder. This script can then be run as a regular python script as described in our -[Python](../../Scientific_Computing/Supported_Applications/Python.md) +[Python](../../Software/Available_Applications/Python.md) documentation. diff --git a/docs/General/FAQs/Ive_run_out_of_storage_space.md b/docs/Getting_Started/FAQs/Ive_run_out_of_storage_space.md similarity index 100% rename from docs/General/FAQs/Ive_run_out_of_storage_space.md rename to docs/Getting_Started/FAQs/Ive_run_out_of_storage_space.md diff --git a/docs/General/FAQs/Login_Troubleshooting.md b/docs/Getting_Started/FAQs/Login_Troubleshooting.md similarity index 100% rename from docs/General/FAQs/Login_Troubleshooting.md rename to docs/Getting_Started/FAQs/Login_Troubleshooting.md diff --git a/docs/General/FAQs/Mahuika_HPC3_Differences.md b/docs/Getting_Started/FAQs/Mahuika_HPC3_Differences.md similarity index 96% rename from docs/General/FAQs/Mahuika_HPC3_Differences.md rename to docs/Getting_Started/FAQs/Mahuika_HPC3_Differences.md index a12eee7c8..70b6fc620 100644 --- a/docs/General/FAQs/Mahuika_HPC3_Differences.md +++ b/docs/Getting_Started/FAQs/Mahuika_HPC3_Differences.md @@ -11,11 +11,11 @@ This article presents an overview comparison of the differences between the NeSI It is not a comprehensive view of the differences and where appropriate individual support pages will be updated to reflect changes and enhancements. For example the [Slurm Reference Sheet](../../Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md) will have a number of changes made to it along with significant changes to the Slurm Partitions. -This page should be read in conjunction with the [Known Issues](../Announcements/Known_Issues_HPC3.md) which are not included here as they are temporary differences to be resolved soon. +This page should be read in conjunction with the [Known Issues](../../Announcements/Known_Issues_HPC3.md) which are not included here as they are temporary differences to be resolved soon. ## Login -We are now using Tuakiri to provide second-factor authentication, and this changes the login experience. See [Standard Terminal Setup HPC3](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) for the full details. +We are now using Tuakiri to provide second-factor authentication, and this changes the login experience. See [Standard Terminal Setup HPC3](../Accessing_the_HPCs/Standard_Terminal_Setup.md) for the full details. ## Operating System @@ -47,7 +47,7 @@ There are snapshots for short-term recovery of deleted files, in `/home/.snapsho ## Access via Web browser -[OnDemand](../../Scientific_Computing/Interactive_computing_with_OnDemand/index.md) has replaced JupyterHub. +[OnDemand](../../Interactive_Computing/OnDemand/index.md) has replaced JupyterHub. OnDemand is more flexible and can deliver more GUI based apps. ## Software diff --git a/docs/General/FAQs/What_Is_A_Trusted_Device.md b/docs/Getting_Started/FAQs/What_Is_A_Trusted_Device.md similarity index 100% rename from docs/General/FAQs/What_Is_A_Trusted_Device.md rename to docs/Getting_Started/FAQs/What_Is_A_Trusted_Device.md diff --git a/docs/General/FAQs/What_are_my-bashrc_and-bash_profile_for.md b/docs/Getting_Started/FAQs/What_are_my-bashrc_and-bash_profile_for.md similarity index 100% rename from docs/General/FAQs/What_are_my-bashrc_and-bash_profile_for.md rename to docs/Getting_Started/FAQs/What_are_my-bashrc_and-bash_profile_for.md diff --git a/docs/General/FAQs/What_does_oom_kill_mean.md b/docs/Getting_Started/FAQs/What_does_oom_kill_mean.md similarity index 100% rename from docs/General/FAQs/What_does_oom_kill_mean.md rename to docs/Getting_Started/FAQs/What_does_oom_kill_mean.md diff --git a/docs/General/FAQs/What_is_Multiple_Factor_Authentication_MFA.md b/docs/Getting_Started/FAQs/What_is_Multiple_Factor_Authentication_MFA.md similarity index 100% rename from docs/General/FAQs/What_is_Multiple_Factor_Authentication_MFA.md rename to docs/Getting_Started/FAQs/What_is_Multiple_Factor_Authentication_MFA.md diff --git a/docs/General/FAQs/What_is_a_core_file.md b/docs/Getting_Started/FAQs/What_is_a_core_file.md similarity index 85% rename from docs/General/FAQs/What_is_a_core_file.md rename to docs/Getting_Started/FAQs/What_is_a_core_file.md index 4c53c4833..0a2658e1b 100644 --- a/docs/General/FAQs/What_is_a_core_file.md +++ b/docs/Getting_Started/FAQs/What_is_a_core_file.md @@ -14,11 +14,11 @@ Your application may crash with an error like, `Segmentation fault (core dumped) These failures are memory-related, such as the program asking for more memory than allocated or for memory it can't legally access. Your first step in troubleshooting should be checking if this is the case, -see [Finding Job_Efficiency](../../Getting_Started/Next_Steps/Finding_Job_Efficiency.md) +see [Finding Job_Efficiency](../../Software/Profiling_and_Debugging/Finding_Job_Efficiency.md) `.core` files are a record of the working memory at time of failure, and can be used for -[debugging](../../Scientific_Computing/Profiling_and_Debugging/Debugging.md). +[debugging](../../Software/Profiling_and_Debugging/Debugging.md). MPI jobs will usually create a `.core` file for each task. The creation of a `.core` file is called a 'core dump' is files is **disabled by default**, diff --git a/docs/General/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md b/docs/Getting_Started/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md similarity index 67% rename from docs/General/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md rename to docs/Getting_Started/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md index 63f5be71a..435500b87 100644 --- a/docs/General/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md +++ b/docs/Getting_Started/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md @@ -13,25 +13,25 @@ use. Examples of software environments on NeSI optimised for data science include: -- [R](../../Scientific_Computing/Supported_Applications/R.md) and - [Python](../../Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md) users +- [R](../../Software/Available_Applications/R.md) and + [Python](../../Software/Available_Applications/TensorFlow_on_GPUs.md) users can get right into using and exploring the several built-in packages or create custom code. -- [Jupyter on NeSI](../../Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md)is +- [Jupyter on NeSI](../../Interactive_Computing/OnDemand/Apps/JupyterLab/index.md) is particularly well suited to artificial intelligence and machine - learning workloads. [RStudio](../../Scientific_Computing/Interactive_computing_with_OnDemand/Apps/RStudio.md) + learning workloads. [RStudio](../../Interactive_Computing/OnDemand/Apps/RStudio.md) and/or Conda can be accessed via Jupyter. - Commonly used data science environments and libraries such as - [Keras](../../Scientific_Computing/Supported_Applications/Keras.md), - [LambdaStack](../../Scientific_Computing/Supported_Applications/Lambda_Stack.md), - [Tensorflow](../../Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md) + [Keras](../../Software/Available_Applications/Keras.md), + [LambdaStack](../../Software/Available_Applications/Lambda_Stack.md), + [Tensorflow](../../Software/Available_Applications/TensorFlow_on_GPUs.md) and [Conda](https://docs.conda.io/en/latest/) are available to create comprehensive workflows. For more information about available software and applications, you -can [browse our catalogue](../../Scientific_Computing/Supported_Applications/index.md). +can [browse our catalogue](../../Software/Available_Applications/index.md). As pictured in the screenshot below, you can type keywords into the catalogue's search field to browse by a specific software name or using diff --git a/docs/General/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md b/docs/Getting_Started/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md similarity index 100% rename from docs/General/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md rename to docs/Getting_Started/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md diff --git a/docs/General/FAQs/Why_am_I_seeing_Account_is_not_ready.md b/docs/Getting_Started/FAQs/Why_am_I_seeing_Account_is_not_ready.md similarity index 100% rename from docs/General/FAQs/Why_am_I_seeing_Account_is_not_ready.md rename to docs/Getting_Started/FAQs/Why_am_I_seeing_Account_is_not_ready.md diff --git a/docs/General/FAQs/Why_does_my_program_crash.md b/docs/Getting_Started/FAQs/Why_does_my_program_crash.md similarity index 91% rename from docs/General/FAQs/Why_does_my_program_crash.md rename to docs/Getting_Started/FAQs/Why_does_my_program_crash.md index 84a34a49e..72ec8a4ff 100644 --- a/docs/General/FAQs/Why_does_my_program_crash.md +++ b/docs/Getting_Started/FAQs/Why_does_my_program_crash.md @@ -11,7 +11,7 @@ investigate. ## OOM One common reason is a limited amount of memory. Then the application -could crash with an [Out Of Memory exception](../../General/FAQs/What_does_oom_kill_mean.md). +could crash with an [Out Of Memory exception](What_does_oom_kill_mean.md). ## Debugger diff --git a/docs/General/FAQs/Why_is_my_job_taking_a_long_time_to_start.md b/docs/Getting_Started/FAQs/Why_is_my_job_taking_a_long_time_to_start.md similarity index 98% rename from docs/General/FAQs/Why_is_my_job_taking_a_long_time_to_start.md rename to docs/Getting_Started/FAQs/Why_is_my_job_taking_a_long_time_to_start.md index 6377f0788..c371527a2 100644 --- a/docs/General/FAQs/Why_is_my_job_taking_a_long_time_to_start.md +++ b/docs/Getting_Started/FAQs/Why_is_my_job_taking_a_long_time_to_start.md @@ -106,7 +106,7 @@ If, compared to other jobs in the queue, your job's priority (third column) and fair share score (fifth column) are both low, this usually means that your project team has recently been using through CPU core hours faster than expected. -See [Fair Share -- How jobs get prioritised](../../Scientific_Computing/Batch_Jobs/Fair_Share.md) for more +See [Fair Share -- How jobs get prioritised](../../Batch_Computing/Fair_Share.md) for more information on Fair Share, how you can check your project's fair share score, and what you can do about a low project fair share score. diff --git a/docs/Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC.md b/docs/Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC.md similarity index 100% rename from docs/Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC.md rename to docs/Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC.md diff --git a/docs/Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md b/docs/Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md similarity index 100% rename from docs/Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md rename to docs/Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md diff --git a/docs/Scientific_Computing/Training/Webinars.md b/docs/Getting_Started/Getting_Help/Webinars.md similarity index 100% rename from docs/Scientific_Computing/Training/Webinars.md rename to docs/Getting_Started/Getting_Help/Webinars.md diff --git a/docs/Scientific_Computing/Training/Workshops.md b/docs/Getting_Started/Getting_Help/Workshops.md similarity index 100% rename from docs/Scientific_Computing/Training/Workshops.md rename to docs/Getting_Started/Getting_Help/Workshops.md diff --git a/docs/Getting_Started/Next_Steps/.pages.yml b/docs/Getting_Started/Next_Steps/.pages.yml deleted file mode 100644 index 094b1e71c..000000000 --- a/docs/Getting_Started/Next_Steps/.pages.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -nav: - - Moving_files_to_and_from_the_cluster.md - - Submitting_your_first_job.md - - Parallel_Execution.md - - Finding_Job_Efficiency.md - - "*" diff --git a/docs/Getting_Started/Next_Steps/The_HPC_environment.md b/docs/Getting_Started/Next_Steps/The_HPC_environment.md deleted file mode 100644 index 9299a758e..000000000 --- a/docs/Getting_Started/Next_Steps/The_HPC_environment.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -created_at: '2019-08-16T01:22:03Z' -tags: [] -vote_count: 0 -vote_sum: 0 -zendesk_article_id: 360001113076 -zendesk_section_id: 360000189716 ---- - -## Environment Modules - -Modules are a convenient  way to provide access to applications  on the cluster. -They prepare the environment you need to run an application. - -For a full list of module commands run `man module` or visit the [lmod documentation](https://lmod.readthedocs.io/en/latest/010_user.html). - -| Command | Description | -|-------------------------------|---------------------------------------------------------------| -| `module spider` | Lists all available modules. (only Mahuika) | -| `module spider [module name]` | Searches available modules for \[module name\] (only Mahuika) | -| `module show [module name]` | Shows information about \[module name\] | -| `module load [module name]` | Loads \[module name\] | -| `module list [module name]` | Lists currently loaded modules. | diff --git a/docs/General/Policy/.pages.yml b/docs/Getting_Started/Policy/.pages.yml similarity index 100% rename from docs/General/Policy/.pages.yml rename to docs/Getting_Started/Policy/.pages.yml diff --git a/docs/General/Policy/Acceptable_Use_Policy.md b/docs/Getting_Started/Policy/Acceptable_Use_Policy.md similarity index 100% rename from docs/General/Policy/Acceptable_Use_Policy.md rename to docs/Getting_Started/Policy/Acceptable_Use_Policy.md diff --git a/docs/General/Policy/Access_Policy.md b/docs/Getting_Started/Policy/Access_Policy.md similarity index 100% rename from docs/General/Policy/Access_Policy.md rename to docs/Getting_Started/Policy/Access_Policy.md diff --git a/docs/General/Policy/Account_Requests_for_non_Tuakiri_Members.md b/docs/Getting_Started/Policy/Account_Requests_for_non_Tuakiri_Members.md similarity index 84% rename from docs/General/Policy/Account_Requests_for_non_Tuakiri_Members.md rename to docs/Getting_Started/Policy/Account_Requests_for_non_Tuakiri_Members.md index 8dbf82595..a1b99d8dc 100644 --- a/docs/General/Policy/Account_Requests_for_non_Tuakiri_Members.md +++ b/docs/Getting_Started/Policy/Account_Requests_for_non_Tuakiri_Members.md @@ -43,6 +43,6 @@ my.nesi.org.nz. If you still can't find the email, {% include "partials/support_request.html" %}. !!! note "What next?" - - [Project Eligibility](../../General/Policy/Allocation_classes.md) - - [Applying for a new project.](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md) - - [Applying to join an existing project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md). + - [Project Eligibility](Allocation_classes.md) + - [Applying for a new project.](../Projects/Applying_for_a_New_Project.md) + - [Applying to join an existing project](../Projects/Applying_to_Join_a_Project.md). diff --git a/docs/General/Policy/Acknowledgement-Citation_and_Publication.md b/docs/Getting_Started/Policy/Acknowledgement-Citation_and_Publication.md similarity index 100% rename from docs/General/Policy/Acknowledgement-Citation_and_Publication.md rename to docs/Getting_Started/Policy/Acknowledgement-Citation_and_Publication.md diff --git a/docs/General/Policy/Allocation_classes.md b/docs/Getting_Started/Policy/Allocation_classes.md similarity index 100% rename from docs/General/Policy/Allocation_classes.md rename to docs/Getting_Started/Policy/Allocation_classes.md diff --git a/docs/General/Policy/Application_Support_Model.md b/docs/Getting_Started/Policy/Application_Support_Model.md similarity index 100% rename from docs/General/Policy/Application_Support_Model.md rename to docs/Getting_Started/Policy/Application_Support_Model.md diff --git a/docs/General/Policy/How_we_review_applications.md b/docs/Getting_Started/Policy/How_we_review_applications.md similarity index 96% rename from docs/General/Policy/How_we_review_applications.md rename to docs/Getting_Started/Policy/How_we_review_applications.md index 6a233f1a0..a943226bf 100644 --- a/docs/General/Policy/How_we_review_applications.md +++ b/docs/Getting_Started/Policy/How_we_review_applications.md @@ -43,8 +43,8 @@ new projects is as follows: research teams. 5. **Decision and notification:** If we approve an initial allocation for your project, we will typically award the project an - [allocation of compute units and also an online storage allocation](../../Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md), - from one of [our allocation classes](../../General/Policy/Allocation_classes.md). + [allocation of compute units and also an online storage allocation](../Allocations/What_is_an_allocation.md), + from one of [our allocation classes](Allocation_classes.md). In an case, we will send you an email telling you about our decision. Our review process for requests for new allocations on existing projects diff --git a/docs/General/Policy/Institutional_allocations.md b/docs/Getting_Started/Policy/Institutional_allocations.md similarity index 78% rename from docs/General/Policy/Institutional_allocations.md rename to docs/Getting_Started/Policy/Institutional_allocations.md index 37f7f8225..80fee228f 100644 --- a/docs/General/Policy/Institutional_allocations.md +++ b/docs/Getting_Started/Policy/Institutional_allocations.md @@ -26,11 +26,11 @@ from your institution. If you are a postgraduate student at a NeSI collaborator, your project will likely be considered for an Institutional allocation rather than a -[Merit](../../General/Policy/Merit_allocations.md) or -[Postgraduate](../../General/Policy/Postgraduate_allocations.md) +[Merit](Merit_allocations.md) or +[Postgraduate](Postgraduate_allocations.md) allocation. -Read more about [how we review applications](../../General/Policy/How_we_review_applications.md). +Read more about [how we review applications](How_we_review_applications.md). To learn more about NeSI Projects or to apply for a new project, please -read our article [Applying for a NeSI Project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md). +read our article [Applying for a NeSI Project](../Projects/Applying_for_a_New_Project.md). diff --git a/docs/General/Policy/Licence_Policy.md b/docs/Getting_Started/Policy/Licence_Policy.md similarity index 100% rename from docs/General/Policy/Licence_Policy.md rename to docs/Getting_Started/Policy/Licence_Policy.md diff --git a/docs/General/Policy/Merit_allocations.md b/docs/Getting_Started/Policy/Merit_allocations.md similarity index 92% rename from docs/General/Policy/Merit_allocations.md rename to docs/Getting_Started/Policy/Merit_allocations.md index f4536a7dd..f8f7304e1 100644 --- a/docs/General/Policy/Merit_allocations.md +++ b/docs/Getting_Started/Policy/Merit_allocations.md @@ -52,7 +52,7 @@ must meet the following criteria: supervisor is a named investigator. Read more about [how we review -applications](../../General/Policy/How_we_review_applications.md). +applications](How_we_review_applications.md). To learn more about REANNZ HPC Projects or to apply for a new project, please -read our article [Applying for a REANNZ HPC Project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md). +read our article [Applying for a REANNZ HPC Project](../Projects/Applying_for_a_New_Project.md). diff --git a/docs/General/Policy/Postgraduate_allocations.md b/docs/Getting_Started/Policy/Postgraduate_allocations.md similarity index 91% rename from docs/General/Policy/Postgraduate_allocations.md rename to docs/Getting_Started/Policy/Postgraduate_allocations.md index d4731733b..fea155ae8 100644 --- a/docs/General/Policy/Postgraduate_allocations.md +++ b/docs/Getting_Started/Policy/Postgraduate_allocations.md @@ -37,7 +37,7 @@ project an allocation from the Postgraduate class: available to meet demand. Read more about [how we review -applications](../../General/Policy/How_we_review_applications.md). +applications](How_we_review_applications.md). To learn more about NeSI Projects, and to apply please review the -content of the section entitled [Applying for a NeSI Project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md). +content of the section entitled [Applying for a NeSI Project](../Projects/Applying_for_a_New_Project.md). diff --git a/docs/General/Policy/Privacy_Policy.md b/docs/Getting_Started/Policy/Privacy_Policy.md similarity index 100% rename from docs/General/Policy/Privacy_Policy.md rename to docs/Getting_Started/Policy/Privacy_Policy.md diff --git a/docs/General/Policy/Proposal_Development_allocations.md b/docs/Getting_Started/Policy/Proposal_Development_allocations.md similarity index 89% rename from docs/General/Policy/Proposal_Development_allocations.md rename to docs/Getting_Started/Policy/Proposal_Development_allocations.md index cb466f023..d9d78db72 100644 --- a/docs/General/Policy/Proposal_Development_allocations.md +++ b/docs/Getting_Started/Policy/Proposal_Development_allocations.md @@ -29,10 +29,10 @@ Proposal Development allocation. Once you have completed your Proposal Development allocation, you are welcome to apply for a further allocation. If you are successful, the project's next allocation will be from another of the -[allocation classes](../../General/Policy/Allocation_classes.md). +[allocation classes](Allocation_classes.md). The [How Applications are Reviewed](How_we_review_applications.md) section provides additional important information for applicants. To learn more about NeSI Projects, and to apply please review the -content of the section entitled [Applying for a NeSI Project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md). +content of the section entitled [Applying for a NeSI Project](../Projects/Applying_for_a_New_Project.md). diff --git a/docs/Getting_Started/Projects/.pages.yml b/docs/Getting_Started/Projects/.pages.yml new file mode 100644 index 000000000..0bff99eea --- /dev/null +++ b/docs/Getting_Started/Projects/.pages.yml @@ -0,0 +1,4 @@ +--- +nav: + - Applying_for_a_New_Project.md + - "*" diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Adding_members_to_your_project.md b/docs/Getting_Started/Projects/Adding_Members_to_your_Project.md similarity index 86% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Adding_members_to_your_project.md rename to docs/Getting_Started/Projects/Adding_Members_to_your_Project.md index 9b556e603..81e0a52f4 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Adding_members_to_your_project.md +++ b/docs/Getting_Started/Projects/Adding_Members_to_your_Project.md @@ -9,8 +9,8 @@ description: How to add a new member to your project. --- !!! prerequisite - - Have a [Account profile](./Creating_an_Account_Profile.md). - - Be the **owner** of a [project](./Applying_for_a_new_project.md). + - Have a [Account profile](../Creating_an_Account.md). + - Be the **owner** of a [project](./Applying_for_a_New_Project.md). 1. Log in to [my.nesi.org.nz](https://my.nesi.org.nz/) via your browser. 2. Under **List Projects**, click on the project you want to add members to. diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md b/docs/Getting_Started/Projects/Applying_for_a_New_Project.md similarity index 88% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md rename to docs/Getting_Started/Projects/Applying_for_a_New_Project.md index 57a47f1d6..932697a39 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md +++ b/docs/Getting_Started/Projects/Applying_for_a_New_Project.md @@ -7,7 +7,7 @@ tags: --- !!! prerequisite - - Have a [Account profile](./Creating_an_Account_Profile.md). + - Have a [Account profile](../Creating_an_Account.md). - NIWA researchers only: read and follow the [NIWA internal documentation for gaining access to the HPCs](https://one.niwa.co.nz/display/ONE/High+Performance+Computing+Facility+Services) (this link is only valid from within the NIWA network or VPN). @@ -21,9 +21,9 @@ tags: - Become familiar with foundational HPC skills, for example by attending a NeSI introductory workshop, one of our [weekly introductory sessions (or watching the - recording)](../../Getting_Started/Getting_Help/Introductory_Material.md), + recording)](../Getting_Help/Introductory_Material.md), or having one or more of your project team members do so. - - Review our [allocation classes](../../General/Policy/Allocation_classes.md). If + - Review our [allocation classes](../Policy/Allocation_classes.md). If you don't think you currently qualify for any class other than Proposal Development, please {% include "partials/support_request.html" %} as soon as possible to discuss your options. Your institution may be in a @@ -65,7 +65,7 @@ information: research programme's current or expected funding) - Details of how your project is funded (this will help determine whether you are eligible for an allocation from our - [Merit](../../General/Policy/Merit_allocations.md) class) + [Merit](../Policy/Merit_allocations.md) class) - Your previous HPC experience - Whether you would like expert scientific programming support on your project @@ -77,7 +77,7 @@ is relevant. !!! prerequisite "What Next?" - Your NeSI Project proposal will be - [reviewed](../../General/Policy/How_we_review_applications.md), + [reviewed](../Policy/How_we_review_applications.md), after which you will be informed of the outcome. - We may contact you if further details are required. - - When your project is approved you will be able to [login for the first time](../../Getting_Started/Accessing_the_HPCs/First_Time_Login.md). + - When your project is approved you will be able to [login for the first time](../Accessing_the_HPCs/First_Time_Login.md). diff --git a/docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md b/docs/Getting_Started/Projects/Applying_to_Join_a_Project.md similarity index 91% rename from docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md rename to docs/Getting_Started/Projects/Applying_to_Join_a_Project.md index 53f4ec4fb..950008182 100644 --- a/docs/Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md +++ b/docs/Getting_Started/Projects/Applying_to_Join_a_Project.md @@ -8,7 +8,7 @@ tags: --- !!! prerequisite - - You must have an [account](./Creating_an_Account_Profile.md). + - You must have an [account](../Creating_an_Account.md). ## How to join a project diff --git a/docs/Getting_Started/my-nesi-org-nz/Logging_in_to_my-nesi-org-nz.md b/docs/Getting_Started/my-nesi-org-nz/Logging_in_to_my-nesi-org-nz.md index d61c081a7..fefd33b94 100644 --- a/docs/Getting_Started/my-nesi-org-nz/Logging_in_to_my-nesi-org-nz.md +++ b/docs/Getting_Started/my-nesi-org-nz/Logging_in_to_my-nesi-org-nz.md @@ -12,7 +12,7 @@ zendesk_section_id: 360001059296 We allow students, academics, alumni and researchers to securely login and create a [NeSI account -profile](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) +profile](../Creating_an_Account.md) using the credentials granted by their home organisation via Tuakiri. ### Tuakiri - federated identity and access management @@ -24,7 +24,7 @@ but many other institutions, including private sector organisations and most central and local government agencies, are not. See also [Creating a NeSI Account -Profile](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) +Profile](../Creating_an_Account.md) ### Support for users outside the Tuakiri federation @@ -34,7 +34,7 @@ profile.](https://my.nesi.org.nz/html/request_nesi_account) NeSI will (if approved) provision a so-called "virtual home account" on Tuakiri. See also [Account Requests for non-Tuakiri -Members](../../General/Policy/Account_Requests_for_non_Tuakiri_Members.md) +Members](../Policy/Account_Requests_for_non_Tuakiri_Members.md) ## Troubleshooting login issues diff --git a/docs/Getting_Started/my-nesi-org-nz/Release_Notes_my-nesi-org-nz/my-nesi-org-nz_release_notes_v2-21-0.md b/docs/Getting_Started/my-nesi-org-nz/Release_Notes_my-nesi-org-nz/my-nesi-org-nz_release_notes_v2-21-0.md index 3173a82f8..ec2f8d713 100644 --- a/docs/Getting_Started/my-nesi-org-nz/Release_Notes_my-nesi-org-nz/my-nesi-org-nz_release_notes_v2-21-0.md +++ b/docs/Getting_Started/my-nesi-org-nz/Release_Notes_my-nesi-org-nz/my-nesi-org-nz_release_notes_v2-21-0.md @@ -20,7 +20,7 @@ search: items under Accounts. - On the Project page and New Allocation Request page, tool tip text referring to - [nn\_corehour\_usage](../../../Scientific_Computing/Batch_Jobs/Checking_resource_usage.md) + [nn\_corehour\_usage](../../../Batch_Computing/Checking_resource_usage.md) will appear when you hover over the Mahuika Compute Units information. diff --git a/docs/Getting_Started/my-nesi-org-nz/Requesting_to_renew_an_allocation_via_my-nesi-org-nz.md b/docs/Getting_Started/my-nesi-org-nz/Requesting_to_renew_an_allocation_via_my-nesi-org-nz.md index 3714cb8b0..b35a4539f 100644 --- a/docs/Getting_Started/my-nesi-org-nz/Requesting_to_renew_an_allocation_via_my-nesi-org-nz.md +++ b/docs/Getting_Started/my-nesi-org-nz/Requesting_to_renew_an_allocation_via_my-nesi-org-nz.md @@ -70,5 +70,5 @@ Please be aware that: - An allocation from an institution's entitlement is subject to approval by that institution. -See [Project Extensions and New Allocations on Existing Projects](../Accounts-Projects_and_Allocations/Allocations_and_Extensions.md) +See [Project Extensions and New Allocations on Existing Projects](../Allocations/Allocations_and_Extensions.md) for more details. diff --git a/docs/Getting_Started/my-nesi-org-nz/The_NeSI_Project_Request_Form.md b/docs/Getting_Started/my-nesi-org-nz/The_NeSI_Project_Request_Form.md index 0c63fb7c3..44f11d7fa 100644 --- a/docs/Getting_Started/my-nesi-org-nz/The_NeSI_Project_Request_Form.md +++ b/docs/Getting_Started/my-nesi-org-nz/The_NeSI_Project_Request_Form.md @@ -8,7 +8,7 @@ zendesk_article_id: 360003648716 zendesk_section_id: 360001059296 --- -See [Applying for a NeSI project](../../Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md)  +See [Applying for a NeSI project](../Projects/Applying_for_a_New_Project.md)  for how to access the form. ## Preparing a request to use NeSI resources diff --git a/docs/Interactive_Computing/.pages.yml b/docs/Interactive_Computing/.pages.yml new file mode 100644 index 000000000..6e9419d27 --- /dev/null +++ b/docs/Interactive_Computing/.pages.yml @@ -0,0 +1,4 @@ +--- +nav: + - Slurm interactive sessions: Slurm_Interactive_Sessions.md + - Interactive computing with OnDemand: OnDemand diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/.pages.yml b/docs/Interactive_Computing/OnDemand/.pages.yml similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/.pages.yml rename to docs/Interactive_Computing/OnDemand/.pages.yml diff --git a/docs/Interactive_Computing/OnDemand/Apps/.pages.yml b/docs/Interactive_Computing/OnDemand/Apps/.pages.yml new file mode 100644 index 000000000..62129ab64 --- /dev/null +++ b/docs/Interactive_Computing/OnDemand/Apps/.pages.yml @@ -0,0 +1,5 @@ +--- +nav: + - VS Code: VSCode.md + - Virtual desktop: virtual_desktop.md + - "*" diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/.pages.yml b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/.pages.yml similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/.pages.yml rename to docs/Interactive_Computing/OnDemand/Apps/JupyterLab/.pages.yml diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md similarity index 96% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md rename to docs/Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md index 48c6e4c60..3413118d2 100644 --- a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md +++ b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md @@ -21,8 +21,8 @@ Python and R kernels by default, which can be selected from the Launcher. Many packages are preinstalled in our default Python and R environments and these can be extended further as described on the -[Python](../../../../Scientific_Computing/Supported_Applications/Python.md) and -[R](../../../../Scientific_Computing/Supported_Applications/R.md) support +[Python](../../../../Software/Available_Applications/Python.md) and +[R](../../../../Software/Available_Applications/R.md) support pages. ## Adding a custom Python kernel @@ -211,7 +211,7 @@ Launcher as "Shared Virtual Env". ## Custom kernel in a Singularity container An example showing setting up a custom kernel running in a Singularity -container can be found on our [Lambda Stack](../../../../Scientific_Computing/Supported_Applications/Lambda_Stack.md#lambda-stack-via-jupyter) +container can be found on our [Lambda Stack](../../../../Software/Available_Applications/Lambda_Stack.md#lambda-stack-via-jupyter) support page. ## Adding a custom R kernel diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md rename to docs/Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/index.md similarity index 97% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md rename to docs/Interactive_Computing/OnDemand/Apps/JupyterLab/index.md index a0e8b0682..f2052402e 100644 --- a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md +++ b/docs/Interactive_Computing/OnDemand/Apps/JupyterLab/index.md @@ -8,7 +8,7 @@ Jupyter allows you to create notebooks that contain live code, equations, visualisations and explanatory text. There are many uses for Jupyter, including data cleaning, analytics and visualisation, machine learning, numerical simulation, managing -[Slurm job submissions](../../../../Getting_Started/Next_Steps/Submitting_your_first_job.md) +[Slurm job submissions](../../../../Batch_Computing/Submitting_your_first_job.md) and workflows and much more. ## Accessing Jupyter on NeSI diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/MATLAB.md b/docs/Interactive_Computing/OnDemand/Apps/MATLAB.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/MATLAB.md rename to docs/Interactive_Computing/OnDemand/Apps/MATLAB.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/RStudio.md b/docs/Interactive_Computing/OnDemand/Apps/RStudio.md similarity index 97% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/RStudio.md rename to docs/Interactive_Computing/OnDemand/Apps/RStudio.md index a1be62296..f6152159f 100644 --- a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/RStudio.md +++ b/docs/Interactive_Computing/OnDemand/Apps/RStudio.md @@ -1,56 +1,56 @@ -# RStudio via OnDemand - - -## Logging in -![UPDATE WITH PROJECT](../../../assets/images/RStudio_via_OOD_on_NeSI_0.png){width=35%} ![](../../../assets/images/RStudio_via_OOD_on_NeSI_1.png){fig.align="right" width=62%} - -## Settings -Recommendation to set *Save Workspace to Never* to avoid saving large files to the workspace. This can be done by going to `Tools` -> `Global Options` -> `General` and setting the `Save workspace to .RData on exit` to `Never`. This will prevent the workspace from being unable to load due to not enough memory in the selected session. - -## Bugs - -### Plots not showing -The current R modules on NeSI OnDemand do not support the default graphics device due to a missing depedency, `cairo`. There is a one off fix for this by changing the backend graphics device from `Default` to `AGG` (Anti-Grain Geometry) in the RStudio settings. - -This can be done by going to `Tools` -> `Global Options` -> `Graphics` and switch `Default` to `AGG`. This will allow the plots to be displayed in the RStudio interface. You do not need to restart the RStudio session for this to take effect. - -![](../../../assets/images/RStudio_via_OOD_on_NeSI_2.png) - -Modules from 4.4 onwards will have this issue fixed. - -### Libraries not showing -There is a bug with the R-Geo and R-bundle-Biocondutor libraries not showing up in the RStudio interface. This is a known issue and is being worked on. There are two workarounds for this issue: - -1. Manually add the library to `.libPaths()` in the R console as shown below: - -```R -myPaths <- .libPaths() -myPaths <- c(myPaths, "/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.2-foss-2023a") - -# reorder paths -myPaths <- c(myPaths[1], myPaths[3], myPaths[2]) - -# reasign the library paths -.libPaths(myPaths) - -# confirm the library paths -.libPaths() -[1] "/nesi/home/$USER/R/foss-2023a/4.3" -[2] "/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.2-foss-2023a" -[3] "/opt/nesi/CS400_centos7_bdw/R/4.3.2-foss-2023a/lib64/R/library" -``` -2. Permanent fix by adding the library path(s) to the `.Rprofile` file in your home directory. This will automatically add the library path to the R console when it starts up. Copy and Paste the following lines to the file: - -``` -# CHECK LIBRARY PATHS -myPaths <- .libPaths() -newPaths <- c("/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.1-gimkl-2022a", - "/opt/nesi/CS400_centos7_bdw/R-bundle-Bioconductor/3.17-gimkl-2022a-R-4.3.1> - -# join the two lists -myPaths <- c(myPaths, newPaths) - -# reassign the library paths -.libPaths(myPaths) -``` -NOTE: Replace the paths with the correct paths for the libraries you want to add. +# RStudio via OnDemand + + +## Logging in +![UPDATE WITH PROJECT](../../../assets/images/RStudio_via_OOD_on_NeSI_0.png){width=35%} ![](../../../assets/images/RStudio_via_OOD_on_NeSI_1.png){fig.align="right" width=62%} + +## Settings +Recommendation to set *Save Workspace to Never* to avoid saving large files to the workspace. This can be done by going to `Tools` -> `Global Options` -> `General` and setting the `Save workspace to .RData on exit` to `Never`. This will prevent the workspace from being unable to load due to not enough memory in the selected session. + +## Bugs + +### Plots not showing +The current R modules on NeSI OnDemand do not support the default graphics device due to a missing depedency, `cairo`. There is a one off fix for this by changing the backend graphics device from `Default` to `AGG` (Anti-Grain Geometry) in the RStudio settings. + +This can be done by going to `Tools` -> `Global Options` -> `Graphics` and switch `Default` to `AGG`. This will allow the plots to be displayed in the RStudio interface. You do not need to restart the RStudio session for this to take effect. + +![](../../../assets/images/RStudio_via_OOD_on_NeSI_2.png) + +Modules from 4.4 onwards will have this issue fixed. + +### Libraries not showing +There is a bug with the R-Geo and R-bundle-Biocondutor libraries not showing up in the RStudio interface. This is a known issue and is being worked on. There are two workarounds for this issue: + +1. Manually add the library to `.libPaths()` in the R console as shown below: + +```R +myPaths <- .libPaths() +myPaths <- c(myPaths, "/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.2-foss-2023a") + +# reorder paths +myPaths <- c(myPaths[1], myPaths[3], myPaths[2]) + +# reasign the library paths +.libPaths(myPaths) + +# confirm the library paths +.libPaths() +[1] "/nesi/home/$USER/R/foss-2023a/4.3" +[2] "/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.2-foss-2023a" +[3] "/opt/nesi/CS400_centos7_bdw/R/4.3.2-foss-2023a/lib64/R/library" +``` +2. Permanent fix by adding the library path(s) to the `.Rprofile` file in your home directory. This will automatically add the library path to the R console when it starts up. Copy and Paste the following lines to the file: + +``` +# CHECK LIBRARY PATHS +myPaths <- .libPaths() +newPaths <- c("/opt/nesi/CS400_centos7_bdw/R-Geo/4.3.1-gimkl-2022a", + "/opt/nesi/CS400_centos7_bdw/R-bundle-Bioconductor/3.17-gimkl-2022a-R-4.3.1> + +# join the two lists +myPaths <- c(myPaths, newPaths) + +# reassign the library paths +.libPaths(myPaths) +``` +NOTE: Replace the paths with the correct paths for the libraries you want to add. diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/VSCode.md b/docs/Interactive_Computing/OnDemand/Apps/VSCode.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/VSCode.md rename to docs/Interactive_Computing/OnDemand/Apps/VSCode.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/virtual_desktop.md b/docs/Interactive_Computing/OnDemand/Apps/virtual_desktop.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/virtual_desktop.md rename to docs/Interactive_Computing/OnDemand/Apps/virtual_desktop.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Release_Notes/index.md b/docs/Interactive_Computing/OnDemand/Release_Notes/index.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/Release_Notes/index.md rename to docs/Interactive_Computing/OnDemand/Release_Notes/index.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/how_to_guide.md b/docs/Interactive_Computing/OnDemand/how_to_guide.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/how_to_guide.md rename to docs/Interactive_Computing/OnDemand/how_to_guide.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/index.md b/docs/Interactive_Computing/OnDemand/index.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/index.md rename to docs/Interactive_Computing/OnDemand/index.md diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/ood_troubleshooting.md b/docs/Interactive_Computing/OnDemand/ood_troubleshooting.md similarity index 100% rename from docs/Scientific_Computing/Interactive_computing_with_OnDemand/ood_troubleshooting.md rename to docs/Interactive_Computing/OnDemand/ood_troubleshooting.md diff --git a/docs/Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md b/docs/Interactive_Computing/Slurm_Interactive_Sessions.md similarity index 99% rename from docs/Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md rename to docs/Interactive_Computing/Slurm_Interactive_Sessions.md index 62cd02a92..e7a827795 100644 --- a/docs/Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md +++ b/docs/Interactive_Computing/Slurm_Interactive_Sessions.md @@ -12,7 +12,7 @@ you to use them interactively as you would the login node. There are two main commands that can be used to make a session, `srun` and `salloc`, both of which use most of the same options available to `sbatch` (see -[our Slurm Reference Sheet](../../Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md)). +[our Slurm Reference Sheet](../Getting_Started/Cheat_Sheets/Slurm-Reference_Sheet.md)). !!! warning An interactive session will, once it starts, use the entire requested diff --git a/docs/NEWPAGE.md b/docs/NEWPAGE.md index 84ec8668d..f5d22050a 100644 --- a/docs/NEWPAGE.md +++ b/docs/NEWPAGE.md @@ -92,6 +92,8 @@ By default, the filename will be use as title of the article/category. Try to keep your title short enough that it does not 'wrap' (become more than one line) in the nav, this usually happens around 24-ish characters however this will vary depending on the letters being used. +Use [Title Case](https://apastyle.apa.org/style-grammar-guidelines/capitalization/title-case), pretty much every word should be capitalised except for the little ones. + !!! tip "File Name hygiene" Regular 'snake_case' naming conventions should be used for articles/categories, i.e. no non-alphanumeric characters (except `_` and `-`). diff --git a/docs/Scientific_Computing/.pages.yml b/docs/Scientific_Computing/.pages.yml deleted file mode 100644 index 9659ae0c1..000000000 --- a/docs/Scientific_Computing/.pages.yml +++ /dev/null @@ -1,10 +0,0 @@ -nav: - - Supported_Applications - - Training - - Interactive_computing_with_OnDemand - - Batch_Jobs - - Profiling_and_Debugging - - HPC_Software_Environment - - Terminal_Setup - - Research_Developer_Cloud - - "*" diff --git a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/.pages.yml b/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/.pages.yml deleted file mode 100644 index f8d7ad5af..000000000 --- a/docs/Scientific_Computing/Interactive_computing_with_OnDemand/Apps/.pages.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -nav: - - JupyterLab: JupyterLab - - RStudio: RStudio.md - - MATLAB: MATLAB.md - - Code server: code_server.md - - Virtual desktop: virtual_desktop.md - - "*" diff --git a/docs/Scientific_Computing/Terminal_Setup/.pages.yml b/docs/Scientific_Computing/Terminal_Setup/.pages.yml deleted file mode 100644 index 961aa69d5..000000000 --- a/docs/Scientific_Computing/Terminal_Setup/.pages.yml +++ /dev/null @@ -1,4 +0,0 @@ -nav: - - Standard_Terminal_Setup.md - - "*" - - X11.md diff --git a/docs/Service_Subscriptions/.pages.yml b/docs/Service_Subscriptions/.pages.yml index cbe183922..74f7dd255 100644 --- a/docs/Service_Subscriptions/.pages.yml +++ b/docs/Service_Subscriptions/.pages.yml @@ -1,5 +1,5 @@ --- nav: - - Contracts_and_billing_processes + - Contracts & Billing: Contracts_and_Billing - Service_Governance - "*" diff --git a/docs/Scientific_Computing/Research_Developer_Cloud/.pages.yml b/docs/Service_Subscriptions/Contracts_and_Billing/.pages.yml similarity index 100% rename from docs/Scientific_Computing/Research_Developer_Cloud/.pages.yml rename to docs/Service_Subscriptions/Contracts_and_Billing/.pages.yml diff --git a/docs/Service_Subscriptions/Contracts_and_billing_processes/Billing_process.md b/docs/Service_Subscriptions/Contracts_and_Billing/Billing_process.md similarity index 100% rename from docs/Service_Subscriptions/Contracts_and_billing_processes/Billing_process.md rename to docs/Service_Subscriptions/Contracts_and_Billing/Billing_process.md diff --git a/docs/Service_Subscriptions/Contracts_and_billing_processes/Types_of_contracts.md b/docs/Service_Subscriptions/Contracts_and_Billing/Types_of_contracts.md similarity index 100% rename from docs/Service_Subscriptions/Contracts_and_billing_processes/Types_of_contracts.md rename to docs/Service_Subscriptions/Contracts_and_Billing/Types_of_contracts.md diff --git a/docs/Service_Subscriptions/Contracts_and_billing_processes/.pages.yml b/docs/Service_Subscriptions/Contracts_and_billing_processes/.pages.yml deleted file mode 100644 index 7a6c3d54c..000000000 --- a/docs/Service_Subscriptions/Contracts_and_billing_processes/.pages.yml +++ /dev/null @@ -1,2 +0,0 @@ -nav: - - "*" diff --git a/docs/Scientific_Computing/Training/.pages.yml b/docs/Service_Subscriptions/Research_Developer_Cloud/.pages.yml similarity index 100% rename from docs/Scientific_Computing/Training/.pages.yml rename to docs/Service_Subscriptions/Research_Developer_Cloud/.pages.yml diff --git a/docs/Scientific_Computing/Research_Developer_Cloud/User_Guides.md b/docs/Service_Subscriptions/Research_Developer_Cloud/User_Guides.md similarity index 99% rename from docs/Scientific_Computing/Research_Developer_Cloud/User_Guides.md rename to docs/Service_Subscriptions/Research_Developer_Cloud/User_Guides.md index 5aa34d7a2..119b5fa47 100644 --- a/docs/Scientific_Computing/Research_Developer_Cloud/User_Guides.md +++ b/docs/Service_Subscriptions/Research_Developer_Cloud/User_Guides.md @@ -15,7 +15,6 @@ Research teams can use this platform to develop novel solutions that enable rese - *Programmable infrastructure:* Applying DevOps practices enabled by Infrastructure as Code (IaC) to automate, measure, collaborate, and learn. - *Partnership-led approaches:* Collaborating with our DevOps specialists to build a platform or tools that can benefit your research community. - ## Features Our platform's cloud building blocks include: diff --git a/docs/Scientific_Computing/HPC_Software_Environment/.pages.yml b/docs/Software/Available_Applications/.pages.yml similarity index 75% rename from docs/Scientific_Computing/HPC_Software_Environment/.pages.yml rename to docs/Software/Available_Applications/.pages.yml index 61d9c36ba..4e35bfd6f 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/.pages.yml +++ b/docs/Software/Available_Applications/.pages.yml @@ -1,2 +1,3 @@ +--- nav: - "*" diff --git a/docs/Scientific_Computing/Supported_Applications/ABAQUS.md b/docs/Software/Available_Applications/ABAQUS.md similarity index 95% rename from docs/Scientific_Computing/Supported_Applications/ABAQUS.md rename to docs/Software/Available_Applications/ABAQUS.md index 21e7a790f..841e76397 100644 --- a/docs/Scientific_Computing/Supported_Applications/ABAQUS.md +++ b/docs/Software/Available_Applications/ABAQUS.md @@ -44,7 +44,7 @@ parameter `academic=TEACHING` or `academic=RESEARCH` in a relevant intuitive formula ⌊ 5 x N0.422 ⌋ where `N` is number of CPUs. -[Hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md) +[Hyperthreading](../Parallel_Computing/Hyperthreading.md) can provide significant speedup to your computations, however hyperthreaded CPUs will use twice the number of licence tokens. It may be worth adding `#SBATCH --hint nomultithread` to your slurm script if @@ -75,7 +75,7 @@ Not all solvers are compatible with all types of parallelisation. === "Serial" For when only one CPU is required, generally as part of - a [job array](../../Getting_Started/Next_Steps/Parallel_Execution.md#job-arrays) + a [job array](../Parallel_Computing/Parallel_Execution.md#job-arrays) ```sl #!/bin/bash -e @@ -186,8 +186,7 @@ loaded with `module load`, you may have to change the compile commands in your l ## Environment file -The [ABAQUS environment -file](http://media.3ds.com/support/simulia/public/v613/installation-and-licensing-guides/books/sgb/default.htm?startat=ch04s01.html) contains +The [ABAQUS environment file](http://media.3ds.com/support/simulia/public/v613/installation-and-licensing-guides/books/sgb/default.htm?startat=ch04s01.html) contains a number of parameters that define how the your job will run, some of these you may with to change. diff --git a/docs/Scientific_Computing/Supported_Applications/ANSYS.md b/docs/Software/Available_Applications/ANSYS.md similarity index 98% rename from docs/Scientific_Computing/Supported_Applications/ANSYS.md rename to docs/Software/Available_Applications/ANSYS.md index 11069f6c6..f1b02469a 100644 --- a/docs/Scientific_Computing/Supported_Applications/ANSYS.md +++ b/docs/Software/Available_Applications/ANSYS.md @@ -142,8 +142,7 @@ the use of variables in what might otherwise be a fixed input. ## Fluent -[Some great documentation on journal -files](https://docs.hpc.shef.ac.uk/en/latest/referenceinfo/ANSYS/fluent/writing-fluent-journal-files.html) +[Some great documentation on journal files](https://docs.hpc.shef.ac.uk/en/latest/referenceinfo/ANSYS/fluent/writing-fluent-journal-files.html) `fluent -help` for a list of commands. @@ -210,8 +209,7 @@ Must have one of these flags. While it will always be more time and resource efficient using a slurm script as shown above, there are occasions where the GUI is required. If you only require a few CPUs for a short while you may run the fluent on -the login node, otherwise use of an [slurm interactive -session](../../Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md) +the login node, otherwise use of an [slurm interactive session](../../Interactive_Computing/Slurm_Interactive_Sessions.md) is recommended. For example. @@ -624,7 +622,7 @@ Progress can be tracked through the GUI as usual. ## ANSYS-Electromagnetic ANSYS-EM jobs can be submitted through a slurm script or by -[interactive session](../../Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md). +[interactive session](../../Interactive_Computing/Slurm_Interactive_Sessions.md). ### RSM diff --git a/docs/Scientific_Computing/Supported_Applications/AlphaFold.md b/docs/Software/Available_Applications/AlphaFold.md similarity index 97% rename from docs/Scientific_Computing/Supported_Applications/AlphaFold.md rename to docs/Software/Available_Applications/AlphaFold.md index 20638b588..9d194cf0b 100644 --- a/docs/Scientific_Computing/Supported_Applications/AlphaFold.md +++ b/docs/Software/Available_Applications/AlphaFold.md @@ -9,11 +9,11 @@ zendesk_section_id: 360000040076 --- -[//]: <> (APPS PAGE BOILERPLATE START) +[//]:AlphaFold.md> (APPS PAGE BOILERPLATE START) {% set app_name = page.title | trim %} {% set app = applications[app_name] %} {% include "partials/app_header.html" %} -[//]: <> (APPS PAGE BOILERPLATE END) +[//]:AlphaFold.md> (APPS PAGE BOILERPLATE END) !!! prerequisite Tips An extended version of AlphaFold2 on NeSI Mahuika cluster which @@ -30,10 +30,10 @@ as AlphaFold throughout the rest of this document. Any publication that discloses findings arising from using this source code or the model parameters -should [cite](https://github.com/deepmind/alphafold#citing-this-work) the [AlphaFold -paper](https://doi.org/10.1038/s41586-021-03819-2). Please also refer to -the [Supplementary -Information](https://static-content.springer.com/esm/art%3A10.1038%2Fs41586-021-03819-2/MediaObjects/41586_2021_3819_MOESM1_ESM.pdf) for +should [cite](https://github.com/deepmind/alphafold#citing-this-work) the +[AlphaFold paper](https://doi.org/10.1038/s41586-021-03819-2). +Please also refer to the [Supplementary +Information](https://static-content.springer.com/esm/art%3A10.1038%2Fs41586-021-03819-2/MediaObjects/41586_2021_3819_MOESM1_ESM.pdf) for a detailed description of the method. Home page is at diff --git a/docs/Scientific_Computing/Supported_Applications/Apptainer.md b/docs/Software/Available_Applications/Apptainer.md similarity index 99% rename from docs/Scientific_Computing/Supported_Applications/Apptainer.md rename to docs/Software/Available_Applications/Apptainer.md index 8d454dba5..004ea31cd 100644 --- a/docs/Scientific_Computing/Supported_Applications/Apptainer.md +++ b/docs/Software/Available_Applications/Apptainer.md @@ -1,8 +1,3 @@ -
- -![apptainer-icon](../../assets/images/apptainer_icon.png) - -
!!! circle-info "The latest version of Apptainer is installed directly on the host operating system of both the login and compute nodes. We recommend using this system-wide version and advise against attempting to load Apptainer via environment modules. Loading the Apptainer module will trigger a message stating: "*The Apptainer environment module has been removed since the system Apptainer is now just as recent*" !!! quote "" diff --git a/docs/Scientific_Computing/Supported_Applications/BLAST.md b/docs/Software/Available_Applications/BLAST.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/BLAST.md rename to docs/Software/Available_Applications/BLAST.md diff --git a/docs/Scientific_Computing/Supported_Applications/BRAKER.md b/docs/Software/Available_Applications/BRAKER.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/BRAKER.md rename to docs/Software/Available_Applications/BRAKER.md diff --git a/docs/Scientific_Computing/Supported_Applications/CESM.md b/docs/Software/Available_Applications/CESM.md similarity index 99% rename from docs/Scientific_Computing/Supported_Applications/CESM.md rename to docs/Software/Available_Applications/CESM.md index 262279dac..2a4360e71 100644 --- a/docs/Scientific_Computing/Supported_Applications/CESM.md +++ b/docs/Software/Available_Applications/CESM.md @@ -30,7 +30,7 @@ both Māui and Mahuika. On Mahuika only, load a module with a more recent version of git than the default one: -``` +```sl module load git ``` diff --git a/docs/Scientific_Computing/Supported_Applications/COMSOL.md b/docs/Software/Available_Applications/COMSOL.md similarity index 98% rename from docs/Scientific_Computing/Supported_Applications/COMSOL.md rename to docs/Software/Available_Applications/COMSOL.md index c5c34f0f2..4b7a9fc49 100644 --- a/docs/Scientific_Computing/Supported_Applications/COMSOL.md +++ b/docs/Software/Available_Applications/COMSOL.md @@ -128,7 +128,7 @@ distribution. ## Interactive Use -Providing you have [set up X11](../Terminal_Setup/X11.md), you can +Providing you have [set up X11](../../Getting_Started/Accessing_the_HPCs/X11.md), you can open the COMSOL GUI by running the command `comsol`. Large jobs should not be run on the login node. diff --git a/docs/Scientific_Computing/Supported_Applications/Clair3.md b/docs/Software/Available_Applications/Clair3.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Clair3.md rename to docs/Software/Available_Applications/Clair3.md diff --git a/docs/Scientific_Computing/Supported_Applications/Cylc.md b/docs/Software/Available_Applications/Cylc.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Cylc.md rename to docs/Software/Available_Applications/Cylc.md diff --git a/docs/Scientific_Computing/Supported_Applications/Delft3D.md b/docs/Software/Available_Applications/Delft3D.md similarity index 93% rename from docs/Scientific_Computing/Supported_Applications/Delft3D.md rename to docs/Software/Available_Applications/Delft3D.md index 362cea043..598c1f718 100644 --- a/docs/Scientific_Computing/Supported_Applications/Delft3D.md +++ b/docs/Software/Available_Applications/Delft3D.md @@ -16,7 +16,8 @@ tags: === "Serial" - For when only one CPU is required, generally as part of a [job array](../../Getting_Started/Next_Steps/Parallel_Execution.md#job-arrays). + For when only one CPU is required, generally as part of a + [job array](../Parallel_Computing/Parallel_Execution.md#job-arrays). ```sl #!/bin/bash -e diff --git a/docs/Scientific_Computing/Supported_Applications/Dorado.md b/docs/Software/Available_Applications/Dorado.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Dorado.md rename to docs/Software/Available_Applications/Dorado.md diff --git a/docs/Scientific_Computing/Supported_Applications/FDS.md b/docs/Software/Available_Applications/FDS.md similarity index 93% rename from docs/Scientific_Computing/Supported_Applications/FDS.md rename to docs/Software/Available_Applications/FDS.md index 919ebea2f..b7723ccdb 100644 --- a/docs/Scientific_Computing/Supported_Applications/FDS.md +++ b/docs/Software/Available_Applications/FDS.md @@ -26,9 +26,9 @@ General documentation can be found [here](https://github.com/firemodels/fds/releases/download/FDS6.7.1/FDS_User_Guide.pdf). FDS can utilise both -[MPI](../../Getting_Started/Next_Steps/Parallel_Execution.md#mpi) +[MPI](../Parallel_Computing/Parallel_Execution.md#mpi) and -[OpenMP](../../Getting_Started/Next_Steps/Parallel_Execution.md#multi-threading) +[OpenMP](../Parallel_Computing/Parallel_Execution.md#multi-threading) ## Example Script diff --git a/docs/Scientific_Computing/Supported_Applications/FlexiBLAS.md b/docs/Software/Available_Applications/FlexiBLAS.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/FlexiBLAS.md rename to docs/Software/Available_Applications/FlexiBLAS.md diff --git a/docs/Scientific_Computing/Supported_Applications/FreeSurfer.md b/docs/Software/Available_Applications/FreeSurfer.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/FreeSurfer.md rename to docs/Software/Available_Applications/FreeSurfer.md diff --git a/docs/Scientific_Computing/Supported_Applications/GATK.md b/docs/Software/Available_Applications/GATK.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/GATK.md rename to docs/Software/Available_Applications/GATK.md diff --git a/docs/Scientific_Computing/Supported_Applications/GROMACS.md b/docs/Software/Available_Applications/GROMACS.md similarity index 97% rename from docs/Scientific_Computing/Supported_Applications/GROMACS.md rename to docs/Software/Available_Applications/GROMACS.md index d713a738b..85f0f86f8 100644 --- a/docs/Scientific_Computing/Supported_Applications/GROMACS.md +++ b/docs/Software/Available_Applications/GROMACS.md @@ -34,7 +34,7 @@ obtained with the Software. === "Serial" For when only one CPU is required, generally as part of - a [job array](../../Getting_Started/Next_Steps/Parallel_Execution.md#job-arrays) + a [job array](../Parallel_Computing/Parallel_Execution.md#job-arrays) ```sl #!/bin/bash -e @@ -83,7 +83,7 @@ obtained with the Software. srun gmx-mpi mdrun-mpi -ntomp ${SLURM_CPUS_PER_TASK} -nomp ${SLURM_NNODES) -s input.tpr -o trajectory.trr -c struct.gro -e energies.edr ``` === "GPU" - For more information on using GPUs see [GPU use on NeSI](../Batch_Jobs/Using_GPUs.md) + For more information on using GPUs see [GPU use on NeSI](../../Batch_Computing/Using_GPUs.md) ```sl #!/bin/bash -e diff --git a/docs/Scientific_Computing/Supported_Applications/GUFI.md b/docs/Software/Available_Applications/GUFI.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/GUFI.md rename to docs/Software/Available_Applications/GUFI.md diff --git a/docs/Scientific_Computing/Supported_Applications/Gaussian.md b/docs/Software/Available_Applications/Gaussian.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Gaussian.md rename to docs/Software/Available_Applications/Gaussian.md diff --git a/docs/Scientific_Computing/Supported_Applications/Java.md b/docs/Software/Available_Applications/Java.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Java.md rename to docs/Software/Available_Applications/Java.md diff --git a/docs/Scientific_Computing/Supported_Applications/Julia.md b/docs/Software/Available_Applications/Julia.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Julia.md rename to docs/Software/Available_Applications/Julia.md diff --git a/docs/Scientific_Computing/Supported_Applications/Keras.md b/docs/Software/Available_Applications/Keras.md similarity index 96% rename from docs/Scientific_Computing/Supported_Applications/Keras.md rename to docs/Software/Available_Applications/Keras.md index f90c18b66..51d1f54bd 100644 --- a/docs/Scientific_Computing/Supported_Applications/Keras.md +++ b/docs/Software/Available_Applications/Keras.md @@ -10,8 +10,8 @@ zendesk_section_id: 360000040076 Keras is a modular and extendable API for building neural networks in Python. Keras is included with TensorFlow. Note that there are -[CPU and](../../Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md) -[GPU versions](../../Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md) of +[CPU and](TensorFlow_on_CPUs.md) +[GPU versions](TensorFlow_on_GPUs.md) of TensorFlow, here we'll use TensorFlow 1.10 for GPUs, which is available as an environment module. diff --git a/docs/Scientific_Computing/Supported_Applications/Lambda_Stack.md b/docs/Software/Available_Applications/Lambda_Stack.md similarity index 95% rename from docs/Scientific_Computing/Supported_Applications/Lambda_Stack.md rename to docs/Software/Available_Applications/Lambda_Stack.md index 8ba695132..b23db19b5 100644 --- a/docs/Scientific_Computing/Supported_Applications/Lambda_Stack.md +++ b/docs/Software/Available_Applications/Lambda_Stack.md @@ -14,13 +14,13 @@ status: [] Stack](https://lambdalabs.com/lambda-stack-deep-learning-software) is an AI software stack from Lambda containing PyTorch, TensorFlow, CUDA, cuDNN and more. On the HPC you can run Lambda Stack via -[Apptainer](../../Scientific_Computing/Supported_Applications/Apptainer.md) (based on the +[Apptainer](Apptainer.md) (based on the official [Dockerfiles](https://github.com/lambdal/lambda-stack-dockerfiles/)). We have provided some pre-built container images (under */opt/nesi/containers/lambda-stack/*) or you can build your own. In the following sections, we will show you how to run Lambda Stack in a Slurm job or interactively via -[JupyterLab](../../Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md). +[JupyterLab](../../Interactive_Computing//OnDemand/Apps/JupyterLab/index.md). You can list the available Lambda Stack version on NeSI by running: @@ -70,7 +70,7 @@ ${CONTAINER} echo "Hello World" The following steps will create a custom Lambda Stack kernel that can be accessed via NeSI's Jupyter service (based on the instructions at -[Jupyter_on_NeSI](../../Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md)). +[Jupyter_on_NeSI](../../Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md)). First, we need to create a kernel definition and wrapper that will launch the container image. Run the following commands on the Mahuika diff --git a/docs/Scientific_Computing/Supported_Applications/MAKER.md b/docs/Software/Available_Applications/MAKER.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/MAKER.md rename to docs/Software/Available_Applications/MAKER.md diff --git a/docs/Scientific_Computing/Supported_Applications/MATLAB.md b/docs/Software/Available_Applications/MATLAB.md similarity index 97% rename from docs/Scientific_Computing/Supported_Applications/MATLAB.md rename to docs/Software/Available_Applications/MATLAB.md index a9f5c62e6..8895ed87d 100644 --- a/docs/Scientific_Computing/Supported_Applications/MATLAB.md +++ b/docs/Software/Available_Applications/MATLAB.md @@ -80,7 +80,7 @@ utilise more than a 4-8 CPUs this way. !!! tip If your code is explicitly parallel at a high level it is preferable to use - [SLURM job arrays](../../Getting_Started/Next_Steps/Parallel_Execution.md) + [SLURM job arrays](../Parallel_Computing/Parallel_Execution.md) as there is less computational overhead and the multiple smaller jobs will queue faster and therefore improve your throughput. @@ -176,7 +176,7 @@ CUDA modules and select the appropriate one. For example, for MATLAB R2021a, use `module load CUDA/11.0.2` before launching MATLAB. If you want to know more about how to access the different type of -available GPUs on NeSI, check the [GPU use on NeSI](../Batch_Jobs/Using_GPUs.md) +available GPUs on NeSI, check the [GPU use on NeSI](../../Batch_Computing/Using_GPUs.md) support page. !!! tip "Support for A100 GPUs" @@ -186,7 +186,7 @@ support page. !!! tip "GPU cost" A GPU device-hour costs more than a core-hour, depending on the type - of GPU. You can find a comparison table in our [What is an allocation?](../../Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md) + of GPU. You can find a comparison table in our [What is an allocation?](../../Getting_Started/Allocations/What_is_an_allocation.md) support page. ### GPU Example diff --git a/docs/Scientific_Computing/Supported_Applications/Miniforge3.md b/docs/Software/Available_Applications/Miniforge3.md similarity index 95% rename from docs/Scientific_Computing/Supported_Applications/Miniforge3.md rename to docs/Software/Available_Applications/Miniforge3.md index 055c47b3b..739c46778 100644 --- a/docs/Scientific_Computing/Supported_Applications/Miniforge3.md +++ b/docs/Software/Available_Applications/Miniforge3.md @@ -7,10 +7,10 @@ tags: !!! note "Preferred Alternatives" - If you want a more reproducible and isolated environment, we - recommend using the [Apptainer containers](../../Scientific_Computing/Supported_Applications/Apptainer.md). + recommend using the [Apptainer containers](Apptainer.md). - If you only need access to Python and standard numerical libraries (numpy, scipy, matplotlib, etc.), you can use the - [Python environment module](../../Scientific_Computing/Supported_Applications/Python.md). + [Python environment module](Python.md). {% set app_name = page.title | trim %} {% set app = applications[app_name] %} diff --git a/docs/Scientific_Computing/Supported_Applications/Molpro.md b/docs/Software/Available_Applications/Molpro.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Molpro.md rename to docs/Software/Available_Applications/Molpro.md diff --git a/docs/Scientific_Computing/Supported_Applications/NWChem.md b/docs/Software/Available_Applications/NWChem.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/NWChem.md rename to docs/Software/Available_Applications/NWChem.md diff --git a/docs/Scientific_Computing/Supported_Applications/ORCA.md b/docs/Software/Available_Applications/ORCA.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/ORCA.md rename to docs/Software/Available_Applications/ORCA.md diff --git a/docs/Scientific_Computing/Supported_Applications/OpenFOAM.md b/docs/Software/Available_Applications/OpenFOAM.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/OpenFOAM.md rename to docs/Software/Available_Applications/OpenFOAM.md diff --git a/docs/Scientific_Computing/Supported_Applications/OpenSees.md b/docs/Software/Available_Applications/OpenSees.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/OpenSees.md rename to docs/Software/Available_Applications/OpenSees.md diff --git a/docs/Scientific_Computing/Supported_Applications/ParaView.md b/docs/Software/Available_Applications/ParaView.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/ParaView.md rename to docs/Software/Available_Applications/ParaView.md diff --git a/docs/Scientific_Computing/Supported_Applications/Python.md b/docs/Software/Available_Applications/Python.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Python.md rename to docs/Software/Available_Applications/Python.md diff --git a/docs/Scientific_Computing/Supported_Applications/R.md b/docs/Software/Available_Applications/R.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/R.md rename to docs/Software/Available_Applications/R.md diff --git a/docs/Scientific_Computing/Supported_Applications/RAxML.md b/docs/Software/Available_Applications/RAxML.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/RAxML.md rename to docs/Software/Available_Applications/RAxML.md diff --git a/docs/Scientific_Computing/Supported_Applications/Relion.md b/docs/Software/Available_Applications/Relion.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Relion.md rename to docs/Software/Available_Applications/Relion.md diff --git a/docs/Scientific_Computing/Supported_Applications/Supernova.md b/docs/Software/Available_Applications/Supernova.md similarity index 97% rename from docs/Scientific_Computing/Supported_Applications/Supernova.md rename to docs/Software/Available_Applications/Supernova.md index 79c3f19fa..0dd579a3e 100644 --- a/docs/Scientific_Computing/Supported_Applications/Supernova.md +++ b/docs/Software/Available_Applications/Supernova.md @@ -126,7 +126,7 @@ takes the following general form `ssh -L :: -N ` - <d> An integer -- <server> see: [Standard Terminal Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) +- <server> see: [Standard Terminal Setup](../../Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md) When details are added to the general form from the specifics in the snippet above, the following could be run.. diff --git a/docs/Scientific_Computing/Supported_Applications/Synda.md b/docs/Software/Available_Applications/Synda.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Synda.md rename to docs/Software/Available_Applications/Synda.md diff --git a/docs/Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md b/docs/Software/Available_Applications/TensorFlow_on_CPUs.md similarity index 95% rename from docs/Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md rename to docs/Software/Available_Applications/TensorFlow_on_CPUs.md index a6915d587..401967121 100644 --- a/docs/Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md +++ b/docs/Software/Available_Applications/TensorFlow_on_CPUs.md @@ -10,7 +10,7 @@ status: deprecated TensorFlow is a popular software library for machine learning applications, see our -[TensorFlow](../../Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md) +[TensorFlow](TensorFlow_on_GPUs.md) article for further information. It is often used with GPUs, as runtimes of the computationally demanding training and inference steps are often shorter compared to multicore CPUs. However, running TensorFlow on CPUs @@ -100,7 +100,7 @@ srun python my_tensorflow_program.py If you are unsure about setting up the memory and runtime parameters, have a look at our article [Ascertaining job -dimensions](../../Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md). +dimensions](../Profiling_and_Debugging/Job_Scaling_Ascertaining_job_dimensions.md). Please also read the section on operator parallelisation below before you choose a number of CPUs. @@ -109,7 +109,7 @@ threading behaviour of the Intel oneDNN library. While these settings should work well for a lot of applications, it is worth trying out different setups (e.g., longer blocktimes) and compare runtimes. Please see our article on [Thread Placement and Thread -Affinity](../../Scientific_Computing/HPC_Software_Environment/Thread_Placement_and_Thread_Affinity.md) +Affinity](../Parallel_Computing/Thread_Placement_and_Thread_Affinity.md) as well as this [Intel article](https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture) for further information and tips for improving peformance on CPUs. diff --git a/docs/Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md b/docs/Software/Available_Applications/TensorFlow_on_GPUs.md similarity index 94% rename from docs/Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md rename to docs/Software/Available_Applications/TensorFlow_on_GPUs.md index ad6095f2a..58932e578 100644 --- a/docs/Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md +++ b/docs/Software/Available_Applications/TensorFlow_on_GPUs.md @@ -20,19 +20,17 @@ running TensorFlow with GPU support. !!! tip "See also" - To request GPU resources using `--gpus-per-node` option of Slurm, - see the [GPU use on - NeSI](../Batch_Jobs/Using_GPUs.md) + see the [GPU use on NeSI](../../Batch_Computing/Using_GPUs.md) documentation page. - To run TensorFlow on CPUs instead, have a look at our article - [TensorFlow on - CPUs](TensorFlow_on_CPUs.md) + [TensorFlow on CPUs](TensorFlow_on_CPUs.md) for tips on how to configure TensorFlow and Slurm for optimal performance. ## Use NeSI modules TensorFlow is available on Mahuika as an -[environment module](../../Getting_Started/Next_Steps/The_HPC_environment.md) +[environment module](index.md) ``` sh module load TensorFlow/2.4.1-gimkl-2020a-Python-3.8.2 @@ -174,7 +172,7 @@ take into consideration the following: You can use containers to run your application on the NeSI platform. We provide support for -[Apptainer](../../Scientific_Computing/Supported_Applications/Apptainer.md) +[Apptainer](Apptainer.md) containers, that can be run by users without requiring additional privileges. Note that Docker containers can be converted into Apptainer containers. @@ -182,16 +180,15 @@ Apptainer containers. For TensorFlow, we recommend using the [official container provided by NVIDIA](https://ngc.nvidia.com/catalog/containers/nvidia:tensorflow). More information about using Apptainer with GPU enabled containers is -available on the [NVIDIA GPU -Containers](../../Scientific_Computing/HPC_Software_Environment/NVIDIA_GPU_Containers.md) +available on the [NVIDIA GPU Containers](../Containers/NVIDIA_GPU_Containers.md) support page. ## Specific versions for A100 Here are the recommended options to run TensorFlow on the A100 GPUs: -- If you use TensorFlow 1, use the TF1 [container provided by - NVIDIA](https://ngc.nvidia.com/catalog/containers/nvidia:tensorflow), +- If you use TensorFlow 1, use the TF1 + [container provided by NVIDIA](https://ngc.nvidia.com/catalog/containers/nvidia:tensorflow), which comes with a version of TensorFlow 1.15 compiled specifically to support the A100 GPUs (Ampere architecture). Other official Python packages won't support the A100, triggering various crashes diff --git a/docs/Scientific_Computing/Supported_Applications/Trinity.md b/docs/Software/Available_Applications/Trinity.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/Trinity.md rename to docs/Software/Available_Applications/Trinity.md diff --git a/docs/Scientific_Computing/Supported_Applications/VASP.md b/docs/Software/Available_Applications/VASP.md similarity index 98% rename from docs/Scientific_Computing/Supported_Applications/VASP.md rename to docs/Software/Available_Applications/VASP.md index 5ef77d082..5fbc9c3c4 100644 --- a/docs/Scientific_Computing/Supported_Applications/VASP.md +++ b/docs/Software/Available_Applications/VASP.md @@ -152,8 +152,8 @@ production simulations. When considering which configuration to use for production you should take into account performance and compute unit cost. -See [Using GPUs](../Batch_Jobs/Using_GPUs.md), for further instructions, and -[Hardware](../Batch_Jobs/Hardware.md#gpgpus) for full GPU specifications. +See [Using GPUs](../../Batch_Computing/Using_GPUs.md), for further instructions, and +[Hardware](../../Batch_Computing/Hardware.md#gpgpus) for full GPU specifications. Some additional notes specific to running VASP on GPUs: diff --git a/docs/Scientific_Computing/Supported_Applications/VTune.md b/docs/Software/Available_Applications/VTune.md similarity index 97% rename from docs/Scientific_Computing/Supported_Applications/VTune.md rename to docs/Software/Available_Applications/VTune.md index 6c2cb17c7..47da03ddd 100644 --- a/docs/Scientific_Computing/Supported_Applications/VTune.md +++ b/docs/Software/Available_Applications/VTune.md @@ -23,7 +23,7 @@ good practice to profile a code before attempting to modify the code to improve its performance. VTune collects key profiling data and presents them in an intuitive way.  Another tool that provides similar information is [ARM -MAP](../../Scientific_Computing/Profiling_and_Debugging/Profiler-ARM_MAP.md). +MAP](../Profiling_and_Debugging/Profiler-ARM_MAP.md). ## How to use VTune diff --git a/docs/Scientific_Computing/Supported_Applications/VirSorter.md b/docs/Software/Available_Applications/VirSorter.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/VirSorter.md rename to docs/Software/Available_Applications/VirSorter.md diff --git a/docs/Scientific_Computing/Supported_Applications/WRF.md b/docs/Software/Available_Applications/WRF.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/WRF.md rename to docs/Software/Available_Applications/WRF.md diff --git a/docs/Scientific_Computing/Supported_Applications/fastStructure.md b/docs/Software/Available_Applications/fastStructure.md similarity index 89% rename from docs/Scientific_Computing/Supported_Applications/fastStructure.md rename to docs/Software/Available_Applications/fastStructure.md index db3991f92..89d4ef9a1 100644 --- a/docs/Scientific_Computing/Supported_Applications/fastStructure.md +++ b/docs/Software/Available_Applications/fastStructure.md @@ -55,4 +55,4 @@ To use this, use the `--format=str` flag and include the file **without** the fi Shout out to a [rather old blog post](https://flowersoftheocean.wordpress.com/2018/04/15/running-faststructure-and-associated-difficulties/) for solving this issue! -The `.str` files output by [ipyrad](../../Scientific_Computing/Supported_Applications/ipyrad.md) should work without issue. Otherwise you may want to convert `.vcf` files to `.bed` files using another tool and proceed with fastStructure using the `.bed` files. +The `.str` files output by [ipyrad](ipyrad.md) should work without issue. Otherwise you may want to convert `.vcf` files to `.bed` files using another tool and proceed with fastStructure using the `.bed` files. diff --git a/docs/Scientific_Computing/Supported_Applications/index.md b/docs/Software/Available_Applications/index.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/index.md rename to docs/Software/Available_Applications/index.md diff --git a/docs/Scientific_Computing/Supported_Applications/ipyrad.md b/docs/Software/Available_Applications/ipyrad.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/ipyrad.md rename to docs/Software/Available_Applications/ipyrad.md diff --git a/docs/Scientific_Computing/Supported_Applications/ont-guppy-gpu.md b/docs/Software/Available_Applications/ont-guppy-gpu.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/ont-guppy-gpu.md rename to docs/Software/Available_Applications/ont-guppy-gpu.md diff --git a/docs/Scientific_Computing/Supported_Applications/snpEff.md b/docs/Software/Available_Applications/snpEff.md similarity index 100% rename from docs/Scientific_Computing/Supported_Applications/snpEff.md rename to docs/Software/Available_Applications/snpEff.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/NVIDIA_GPU_Containers.md b/docs/Software/Containers/NVIDIA_GPU_Containers.md similarity index 95% rename from docs/Scientific_Computing/HPC_Software_Environment/NVIDIA_GPU_Containers.md rename to docs/Software/Containers/NVIDIA_GPU_Containers.md index cd152daad..5e771ba22 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/NVIDIA_GPU_Containers.md +++ b/docs/Software/Containers/NVIDIA_GPU_Containers.md @@ -46,8 +46,7 @@ running the NAMD image on NeSI, based on the NVIDIA instructions directly, which does not require root access: !!! note - Please do refer [Build Environment - Variables](../../Scientific_Computing/Supported_Applications/Apptainer.md) + Please do refer [Build Environment Variables](../../Software/Available_Applications/Apptainer.md) prior to running the following `pull` command. ```sh diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_in_parallel.md b/docs/Software/Containers/Run_an_executable_under_Apptainer_in_parallel.md similarity index 100% rename from docs/Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_in_parallel.md rename to docs/Software/Containers/Run_an_executable_under_Apptainer_in_parallel.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_on_gpu.md b/docs/Software/Containers/Run_an_executable_under_Apptainer_on_gpu.md similarity index 100% rename from docs/Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_on_gpu.md rename to docs/Software/Containers/Run_an_executable_under_Apptainer_on_gpu.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Installing_Applications_Yourself.md b/docs/Software/Installing_Applications_Yourself.md similarity index 94% rename from docs/Scientific_Computing/HPC_Software_Environment/Installing_Applications_Yourself.md rename to docs/Software/Installing_Applications_Yourself.md index 303cb43c3..3842b6cec 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/Installing_Applications_Yourself.md +++ b/docs/Software/Installing_Applications_Yourself.md @@ -8,7 +8,7 @@ tags: Before installing your own applications, first check; - The software you want is not already installed. `module spider ` can be used to search software, -or see [Supported Applications](../Supported_Applications/index.md). +or see [Supported Applications](Available_Applications/index.md). - If you are looking for a new version of existing software, {% include "partials/support_request.html" %} and we will install the new version. - If you would like us to install something for you or help you install something yourself {% include "partials/support_request.html" %}. If the software is popular, We may decide to install it centrally, in which case there will be no additional steps for you. Otherwise the software will be installed in your project directory, in which case it is your responsibility to maintain. @@ -22,13 +22,13 @@ See [Software Installation Request](Software_Installation_Request.md) for guidel How to add package to an existing module will vary based on the language in question. -- [Python](../Supported_Applications/Python.md#python-packages) -- [R](../Supported_Applications/R.md#dealing-with-packages) -- [Julia](../Supported_Applications/Julia.md#installing-julia-packages) -- [MATLAB](../Supported_Applications/MATLAB.md#adding-support-packages) +- [Python](Available_Applications/Python.md#python-packages) +- [R](Available_Applications/R.md#dealing-with-packages) +- [Julia](Available_Applications/Julia.md#installing-julia-packages) +- [MATLAB](Available_Applications/MATLAB.md#adding-support-packages) For other languages check if we have additional documentation for it -in our [application documentation](../Supported_Applications/index.md). +in our [application documentation](Available_Applications/index.md). ## Other Applications diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Configuring_Dask_MPI_jobs.md b/docs/Software/Parallel_Computing/Configuring_Dask_MPI_jobs.md similarity index 97% rename from docs/Scientific_Computing/HPC_Software_Environment/Configuring_Dask_MPI_jobs.md rename to docs/Software/Parallel_Computing/Configuring_Dask_MPI_jobs.md index 248ddb112..2e64ca6ee 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/Configuring_Dask_MPI_jobs.md +++ b/docs/Software/Parallel_Computing/Configuring_Dask_MPI_jobs.md @@ -78,7 +78,7 @@ dependencies: !!! info "See also" See the - [Miniforge3](../../Scientific_Computing/Supported_Applications/Miniforge3.md) + [Miniforge3](../Available_Applications/Miniforge3.md) page for more information on how to create and manage Miniconda environments on NeSI. @@ -97,7 +97,7 @@ then assigns different roles to the different ranks: This implies that **Dask-MPI jobs must be launched on at least 3 MPI ranks!** Ranks 0 and 1 often perform much less work than the other ranks, it can therefore be beneficial to use -[Hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md) +[Hyperthreading](Hyperthreading.md) to place these two ranks onto a single physical core. Ensure that activating hyperthreading does not slow down the worker ranks by running a short test workload with and without hyperthreading. @@ -261,7 +261,7 @@ where the `%runscript` section ensures that the Python script passed to Conda environment inside the container. !!! note Tips - You can build this container on NeSI,following the instructions from the [dedicated supportpage](../Supported_Applications/Apptainer.md) + You can build this container on NeSI,following the instructions from the [dedicated supportpage](../Available_Applications/Apptainer.md) ### Slurm configuration diff --git a/docs/Scientific_Computing/Batch_Jobs/Hyperthreading.md b/docs/Software/Parallel_Computing/Hyperthreading.md similarity index 98% rename from docs/Scientific_Computing/Batch_Jobs/Hyperthreading.md rename to docs/Software/Parallel_Computing/Hyperthreading.md index d9ff5ec42..a3a1032e3 100644 --- a/docs/Scientific_Computing/Batch_Jobs/Hyperthreading.md +++ b/docs/Software/Parallel_Computing/Hyperthreading.md @@ -34,7 +34,7 @@ once your job starts you will have twice the number of CPUs as `ntasks`. If you set `--cpus-per-task=n`, Slurm will request `n` logical CPUs per task, i.e., will set `n` threads for the job. Your code must be capable of running Hyperthreaded (for example using -[OpenMP](../../Scientific_Computing/HPC_Software_Environment/OpenMP_settings.md)) +[OpenMP](OpenMP_settings.md)) if `--cpus-per-task > 1`. Setting `--hint=nomultithread` with `srun` or `sbatch` causes Slurm to @@ -187,7 +187,7 @@ considered a bonus. for MPI jobs that request the same number of tasks on every node, we recommend to specify `--mem` (i.e. memory per node) instead. See [How to request memory - (RAM)](../../General/FAQs/How_do_I_request_memory.md) for more + (RAM)](../../Getting_Started/FAQs/How_do_I_request_memory.md) for more information. - Non-MPI jobs which specify `--cpus-per-task` and use **srun** should also set `--ntasks=1`, otherwise the program will be run twice in diff --git a/docs/Getting_Started/Next_Steps/MPI_Scaling_Example.md b/docs/Software/Parallel_Computing/MPI_Scaling_Example.md similarity index 99% rename from docs/Getting_Started/Next_Steps/MPI_Scaling_Example.md rename to docs/Software/Parallel_Computing/MPI_Scaling_Example.md index 7181b25c0..79a36b83b 100644 --- a/docs/Getting_Started/Next_Steps/MPI_Scaling_Example.md +++ b/docs/Software/Parallel_Computing/MPI_Scaling_Example.md @@ -174,7 +174,7 @@ Let's run our Slurm script with sbatch and look at our output from Our job performed 5,000 seeds using 2 physical CPU cores (each MPI task will always receive 2 logical CPUs which is equal to 1 physical CPUs. For a more in depth explanation about logical and physical CPU cores see -our [Hyperthreading article](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md)) +our [Hyperthreading article](Hyperthreading.md)) and a maximum memory of 166,744KB (0.16 GB). In total, the job ran for 18 minutes and 51 seconds. diff --git a/docs/Getting_Started/Next_Steps/Multithreading_Scaling_Example.md b/docs/Software/Parallel_Computing/Multithreading_Scaling_Example.md similarity index 99% rename from docs/Getting_Started/Next_Steps/Multithreading_Scaling_Example.md rename to docs/Software/Parallel_Computing/Multithreading_Scaling_Example.md index a82359307..09c1ea98d 100644 --- a/docs/Getting_Started/Next_Steps/Multithreading_Scaling_Example.md +++ b/docs/Software/Parallel_Computing/Multithreading_Scaling_Example.md @@ -274,4 +274,4 @@ memory as we may otherwise have run out. about 20% more wall time and memory than you think you are going to need to minimise the chance of your jobs failing due to a lack of resources. Your project's fair share score considers the time actually used by the - job, not the time requested by the job. \ No newline at end of file + job, not the time requested by the job. diff --git a/docs/Scientific_Computing/HPC_Software_Environment/OpenMP_settings.md b/docs/Software/Parallel_Computing/OpenMP_settings.md similarity index 90% rename from docs/Scientific_Computing/HPC_Software_Environment/OpenMP_settings.md rename to docs/Software/Parallel_Computing/OpenMP_settings.md index 357b33915..5ebd21330 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/OpenMP_settings.md +++ b/docs/Software/Parallel_Computing/OpenMP_settings.md @@ -20,17 +20,17 @@ all that is necessary to get 16 OpenMP threads is: in your Slurm script - although this can sometimes be more complicated, e.g., with -[TensorFlow on CPUs](../../Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md). +[TensorFlow on CPUs](../Available_Applications/TensorFlow_on_CPUs.md). In order to achieve good and consistent parallel scaling, additional settings may be required. This is particularly true on Mahuika where nodes are generally shared between different Slurm jobs. Following are some settings that can help improve scaling and/or make your timings more consistent, additional information can be found in our article -[Thread Placement and Thread Affinity](../../Scientific_Computing/HPC_Software_Environment/Thread_Placement_and_Thread_Affinity.md). +[Thread Placement and Thread Affinity](./Thread_Placement_and_Thread_Affinity.md). 1. `--threads-per-core=2`. Use this option to tell srun or sbatch to -that you want to use [Hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md), +that you want to use [Hyperthreading](Hyperthreading.md), so use both of the virual CPUs available on each physical core, halving the number of physical cores you occupy. If you use hyperthreading, you will be charged for the number of physical cores that diff --git a/docs/Getting_Started/Next_Steps/Parallel_Execution.md b/docs/Software/Parallel_Computing/Parallel_Execution.md similarity index 98% rename from docs/Getting_Started/Next_Steps/Parallel_Execution.md rename to docs/Software/Parallel_Computing/Parallel_Execution.md index a76d16e71..f9fe0cb63 100644 --- a/docs/Getting_Started/Next_Steps/Parallel_Execution.md +++ b/docs/Software/Parallel_Computing/Parallel_Execution.md @@ -15,7 +15,7 @@ The are three types of parallel execution we will cover are [Multi-Threading](# - `--mem-per-cpu=512MB` will give 512 MB of RAM per *logical* core. - If `--hint=nomultithread` is used then `--cpus-per-task` will now refer to physical cores, but `--mem-per-cpu=512MB` still refers to logical cores. -See [our article on hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md) for more information. +See [our article on hyperthreading](Hyperthreading.md) for more information. ## Multi-threading diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Thread_Placement_and_Thread_Affinity.md b/docs/Software/Parallel_Computing/Thread_Placement_and_Thread_Affinity.md similarity index 99% rename from docs/Scientific_Computing/HPC_Software_Environment/Thread_Placement_and_Thread_Affinity.md rename to docs/Software/Parallel_Computing/Thread_Placement_and_Thread_Affinity.md index a9e908532..f4a45c0c5 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/Thread_Placement_and_Thread_Affinity.md +++ b/docs/Software/Parallel_Computing/Thread_Placement_and_Thread_Affinity.md @@ -8,7 +8,7 @@ status: deprecated Multithreading with OpenMP and other threading libraries is an important way to parallelise scientific software for faster execution (see our article on [Parallel -Execution](../../Getting_Started/Next_Steps/Parallel_Execution.md) for +Execution](Parallel_Execution.md) for an introduction). Care needs to be taken when running multiple threads on the HPC to achieve best performance - getting it wrong can easily increase compute times by tens of percents, sometimes even more. This is @@ -34,7 +34,7 @@ performance, as a socket connects the processor to its RAM and other processors. A processor in each socket consists of multiple physical cores, and each physical core is split into two logical cores using a technology called -[Hyperthreading](../../Scientific_Computing/Batch_Jobs/Hyperthreading.md)). +[Hyperthreading](Hyperthreading.md)). A processor also includes caches - a [cache](https://en.wikipedia.org/wiki/CPU_cache) is very fast memory diff --git a/docs/Scientific_Computing/Profiling_and_Debugging/.pages.yml b/docs/Software/Profiling_and_Debugging/.pages.yml similarity index 100% rename from docs/Scientific_Computing/Profiling_and_Debugging/.pages.yml rename to docs/Software/Profiling_and_Debugging/.pages.yml diff --git a/docs/Scientific_Computing/Profiling_and_Debugging/Debugging.md b/docs/Software/Profiling_and_Debugging/Debugging.md similarity index 100% rename from docs/Scientific_Computing/Profiling_and_Debugging/Debugging.md rename to docs/Software/Profiling_and_Debugging/Debugging.md diff --git a/docs/Getting_Started/Next_Steps/Finding_Job_Efficiency.md b/docs/Software/Profiling_and_Debugging/Finding_Job_Efficiency.md similarity index 97% rename from docs/Getting_Started/Next_Steps/Finding_Job_Efficiency.md rename to docs/Software/Profiling_and_Debugging/Finding_Job_Efficiency.md index fae8acd70..5ffa6a34e 100644 --- a/docs/Getting_Started/Next_Steps/Finding_Job_Efficiency.md +++ b/docs/Software/Profiling_and_Debugging/Finding_Job_Efficiency.md @@ -182,7 +182,7 @@ time* the CPUs are in use. This is not enough to get a picture of overall job efficiency, as required CPU time *may vary by number of CPU*s. -The only way to get the full context, is to compare walltime performance between jobs at different scale. See [Job Scaling](../../Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md) for more details. +The only way to get the full context, is to compare walltime performance between jobs at different scale. See [Job Scaling](Job_Scaling_Ascertaining_job_dimensions.md) for more details. ### Example diff --git a/docs/Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md b/docs/Software/Profiling_and_Debugging/Job_Scaling_Ascertaining_job_dimensions.md similarity index 89% rename from docs/Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md rename to docs/Software/Profiling_and_Debugging/Job_Scaling_Ascertaining_job_dimensions.md index 8cf4e5379..9852b7fdd 100644 --- a/docs/Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md +++ b/docs/Software/Profiling_and_Debugging/Job_Scaling_Ascertaining_job_dimensions.md @@ -31,7 +31,7 @@ ascertain how much of each of these resources you will need. Asking for too little or too much, however, can both cause problems: your jobs will be at increased risk of taking a long time in the queue or failing, and -your project's [fair share score](../../Scientific_Computing/Batch_Jobs/Fair_Share.md) +your project's [fair share score](../../Batch_Computing/Fair_Share.md) is likely to suffer. Your project's fair share score will be reduced in view of compute time spent regardless of whether you obtain a result or @@ -43,7 +43,7 @@ not. | Memory | The job may wait in the queue for longer. Your fair share score will fall more than necessary. | Your job will fail, probably with an 'OUT OF MEMORY' error, segmentation fault or bus error. This may not happen immediately. | | Wall time | The job may wait in the queue for longer than necessary | The job will run out of time and get killed. | -***See [What is an allocation?](../../Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md) for more details on how each resource effects your compute usage.*** +***See [What is an allocation?](../../Getting_Started/Allocations/What_is_an_allocation.md) for more details on how each resource effects your compute usage.*** It is therefore important to try and make your jobs resource requests reasonably accurate. In this article we will discuss how you can scale @@ -74,8 +74,8 @@ will not have waited for hours or days in the queue beforehand. !!! example - - [Multithreading Scaling](../../Getting_Started/Next_Steps/Multithreading_Scaling_Example.md) - - [MPI Scaling](../../Getting_Started/Next_Steps/MPI_Scaling_Example.md) + - [Multithreading Scaling](../../Software/Parallel_Computing/Multithreading_Scaling_Example.md) + - [MPI Scaling](../../Software/Parallel_Computing/MPI_Scaling_Example.md) !!! tip "Webinar: How to estimate CPU, memory & time needs" diff --git a/docs/Scientific_Computing/Profiling_and_Debugging/Profiler-ARM_MAP.md b/docs/Software/Profiling_and_Debugging/Profiler-ARM_MAP.md similarity index 100% rename from docs/Scientific_Computing/Profiling_and_Debugging/Profiler-ARM_MAP.md rename to docs/Software/Profiling_and_Debugging/Profiler-ARM_MAP.md diff --git a/docs/Scientific_Computing/Profiling_and_Debugging/Profiler-VTune.md b/docs/Software/Profiling_and_Debugging/Profiler-VTune.md similarity index 100% rename from docs/Scientific_Computing/Profiling_and_Debugging/Profiler-VTune.md rename to docs/Software/Profiling_and_Debugging/Profiler-VTune.md diff --git a/docs/Scientific_Computing/Profiling_and_Debugging/Slurm_Native_Profiling.md b/docs/Software/Profiling_and_Debugging/Slurm_Native_Profiling.md similarity index 100% rename from docs/Scientific_Computing/Profiling_and_Debugging/Slurm_Native_Profiling.md rename to docs/Software/Profiling_and_Debugging/Slurm_Native_Profiling.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Software_Installation_Request.md b/docs/Software/Software_Installation_Request.md similarity index 100% rename from docs/Scientific_Computing/HPC_Software_Environment/Software_Installation_Request.md rename to docs/Software/Software_Installation_Request.md diff --git a/docs/Scientific_Computing/HPC_Software_Environment/Software_Version_Management.md b/docs/Software/Software_Version_Management.md similarity index 84% rename from docs/Scientific_Computing/HPC_Software_Environment/Software_Version_Management.md rename to docs/Software/Software_Version_Management.md index 8e59049d3..302be2c50 100644 --- a/docs/Scientific_Computing/HPC_Software_Environment/Software_Version_Management.md +++ b/docs/Software/Software_Version_Management.md @@ -4,15 +4,11 @@ tags: - software - versions title: Software Version Management -vote_count: 0 -vote_sum: 0 -zendesk_article_id: 360001045096 -zendesk_section_id: 360000040056 --- Much of the software installed on the NeSI cluster have multiple versions available as shown on the -[supported applications page](../Supported_Applications/index.md) +[supported applications page](Available_Applications/index.md) or by using the `module avail` or `module spider` commands. If only the application name is given a default version will be chosen, diff --git a/docs/Storage/Data_Transfer_Services/Data_Transfer_using_Globus.md b/docs/Storage/Data_Transfer_Services/Data_Transfer_using_Globus.md index 7f7f6780e..249f36052 100644 --- a/docs/Storage/Data_Transfer_Services/Data_Transfer_using_Globus.md +++ b/docs/Storage/Data_Transfer_Services/Data_Transfer_using_Globus.md @@ -23,7 +23,7 @@ To use Globus to transfer data to/from NeSI platforms, you need: 1. A Globus account (see [Initial Globus Sign-Up and Globus ID](../../Storage/Data_Transfer_Services/Initial_Globus_Sign_Up-and_your_Globus_Identities.md)) 2. An active NeSI account (see - [Creating a NeSI Account](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md)) + [Creating a NeSI Account](../../Getting_Started/Creating_an_Account.md)) 3. Access privileges to the Globus endpoint/collection you plan on transferring data from or to. This endpoint/collection could be a personal one on your workstation, or it could be managed diff --git a/docs/Storage/Data_Transfer_Services/Globus_Quick_Start_Guide.md b/docs/Storage/Data_Transfer_Services/Globus_Quick_Start_Guide.md index 810bd5c5b..0b17cf951 100644 --- a/docs/Storage/Data_Transfer_Services/Globus_Quick_Start_Guide.md +++ b/docs/Storage/Data_Transfer_Services/Globus_Quick_Start_Guide.md @@ -11,7 +11,7 @@ between two Globus Data Transfer Nodes (DTNs). To use Globus to transfer data to or from NeSI, you need: 1. An active NeSI account (see - [Creating a NeSI Account](../../Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md)) + [Creating a NeSI Account](../../Getting_Started/Creating_an_Account.md)) 2. A Globus account (see [Initial Globus Sign-Up and Globus ID](../../Storage/Data_Transfer_Services/Initial_Globus_Sign_Up-and_your_Globus_Identities.md)) 3. Access to Globus DTNs or endpoints diff --git a/docs/Storage/File_Systems_and_Quotas/File_permissions_and_groups.md b/docs/Storage/File_Systems_and_Quotas/File_permissions_and_groups.md index 531076bdf..0c59234e7 100644 --- a/docs/Storage/File_Systems_and_Quotas/File_permissions_and_groups.md +++ b/docs/Storage/File_Systems_and_Quotas/File_permissions_and_groups.md @@ -84,7 +84,7 @@ project group. directory will inherit neither the group nor the setgid bit. You probably don't want this to happen. For instructions on how to prevent it, please see our article: - [How can I let my fellow project team members read or write my files?](../../General/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md) + [How can I let my fellow project team members read or write my files?](../../Getting_Started/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md) By default, the world, i.e. people not in the project team, have no privileges in respect of a project directory, with certain exceptions. @@ -139,6 +139,6 @@ If we agree to set up a special-purpose directory for you, we will discuss and a suitable permissions model. !!! prerequisite "See also" - - [How can I let my fellow project team members read or write my files?](../../General/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md) - - [How can I give read-only team members access to my files?](../../General/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md) + - [How can I let my fellow project team members read or write my files?](../../Getting_Started/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md) + - [How can I give read-only team members access to my files?](../../Getting_Started/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md) - [filesystems and quotas](./Filesystems_and_Quotas.md) diff --git a/docs/Storage/File_Systems_and_Quotas/Filesystems_and_Quotas.md b/docs/Storage/File_Systems_and_Quotas/Filesystems_and_Quotas.md index 1a16aa914..376e9003f 100644 --- a/docs/Storage/File_Systems_and_Quotas/Filesystems_and_Quotas.md +++ b/docs/Storage/File_Systems_and_Quotas/Filesystems_and_Quotas.md @@ -102,7 +102,8 @@ apply per-project disk space quotas to projects on this filesystem. The default per-project quotas are as described in the above table; if you require more temporary (scratch) space for your project than the default quota allows for, you can discuss your -requirements with us during [the project application process](../../General/Policy/How_we_review_applications.md), +requirements with us during +[the project application process](../../Getting_Started/Policy/How_we_review_applications.md), or {% include "partials/support_request.html" %} at any time. To ensure this filesystem remains fit-for-purpose, we have a regular @@ -125,7 +126,8 @@ an Automatic Tape Library (ATL). Files will remain on Freezer temporarily, typically for hours to days, before being moved to tape. A catalogue of files on tape will remain on the disk for quick access. -See more information about the long term storage see our [documentation about the Freezer storage service](../Long_Term_Storage/Freezer_long_term_storage.md). +See more information about the long term storage see our +[documentation about the Freezer storage service](../Long_Term_Storage/Freezer_long_term_storage.md). ## Snapshots diff --git a/docs/Storage/Long_Term_Storage/.pages.yml b/docs/Storage/Long_Term_Storage/.pages.yml index 57308d3dc..1b9a2fb8e 100644 --- a/docs/Storage/Long_Term_Storage/.pages.yml +++ b/docs/Storage/Long_Term_Storage/.pages.yml @@ -3,6 +3,5 @@ nav: - Configuring_s3cmd.md - Freezer_Guide.md - Other_Useful_Commands.md - - Troubleshooting.md - "*" - Release Notes freezer.nesi.org.nz: Release_Notes_freezer-nesi-org-nz diff --git a/docs/Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md b/docs/Storage/Moving_files_to_and_from_the_cluster.md similarity index 80% rename from docs/Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md rename to docs/Storage/Moving_files_to_and_from_the_cluster.md index feb602d83..e91edbbb2 100644 --- a/docs/Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md +++ b/docs/Storage/Moving_files_to_and_from_the_cluster.md @@ -8,19 +8,19 @@ tags: --- !!! prerequisite - Have an [active account and project.](../Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md) + Have an [active account and project.](../Getting_Started/Creating_an_Account.md) -Find more information on [our filesystem](../../Storage/File_Systems_and_Quotas/Filesystems_and_Quotas.md). +Find more information on [our filesystem](./File_Systems_and_Quotas/Filesystems_and_Quotas.md). ## OnDemand Requiring only a web browser, the instructions are same whether your are connecting from a Windows, Mac or a Linux computer. -See [OnDemand how to guide](../../Scientific_Computing/Interactive_computing_with_OnDemand/how_to_guide.md) for more info. +See [OnDemand how to guide](../Interactive_Computing/OnDemand/how_to_guide.md) for more info. ## Standard Terminal !!! prerequisite - Have SSH setup as described in [Standard Terminal Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) + Have SSH setup as described in [Standard Terminal Setup](../Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md) In a local terminal the following commands can be used to: @@ -38,7 +38,7 @@ scp mahuika: !!! note - This will only work if you have set up aliases as described in - [Terminal Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md). + [Terminal Setup](../Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md). - As the term 'mahuika' is defined locally, the above commands *only works when using a local terminal* (i.e. not on Mahuika). - If you are using Windows subsystem, the root paths are different @@ -54,7 +54,7 @@ your password. ## File Managers !!! prerequisite - Have SSH setup as described in [Standard Terminal Setup](../../Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md) + Have SSH setup as described in [Standard Terminal Setup](../Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md) Most file managers can be used to connect to a remote directory simply by typing in the address bar provided your have an active connection to @@ -67,22 +67,22 @@ This **does not** work for File Explorer (Windows default) This **does not** work for Finder (Mac default) -![files](../../assets/images/Moving_files_to_and_from_the_cluster_1.png) +![files](../assets/images/Moving_files_to_and_from_the_cluster_1.png) If your default file manager does not support mounting over SFTP, see -[Can I use SSHFS to mount the cluster filesystem on my local machine?](../../General/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md). +[Can I use SSHFS to mount the cluster filesystem on my local machine?](../Getting_Started/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md). ## MobaXterm !!! prerequisite - [MobaXterm Setup Windows](../../Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md) + [MobaXterm Setup Windows](../Getting_Started/Accessing_the_HPCs/MobaXterm_Setup_Windows.md) See [Standard Terminal]](Moving_files_to_and_from_the_cluster.md#standard-terminal), [Rclone]](Moving_files_to_and_from_the_cluster.md#rclone), or [Rsync]](Moving_files_to_and_from_the_cluster.md#rsync) for information on how to move files to and from the HPC in the terminal. ## WinSCP !!! prerequisite - [WinSCP-PuTTY Setup Windows](../../Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md) + [WinSCP-PuTTY Setup Windows](../Getting_Started/Accessing_the_HPCs/WinSCP-PuTTY_Setup_Windows.md) As WinSCP uses multiple tunnels for file transfer you will be required to authenticate again on your first file operation of the session. The @@ -94,7 +94,7 @@ authentication. Globus is available for those with large amounts of data, security concerns, or connection consistency issues. You can find more details in -[Data_Transfer_using_Globus](../../Storage/Data_Transfer_Services/Data_Transfer_using_Globus.md). +[Data_Transfer_using_Globus](./Data_Transfer_Services/Data_Transfer_using_Globus.md). ## Rclone diff --git a/docs/redirect_map.yml b/docs/redirect_map.yml index 4a182bf99..a91061939 100644 --- a/docs/redirect_map.yml +++ b/docs/redirect_map.yml @@ -1,20 +1,170 @@ -General/FAQs/Two_Factor_Authentication_FAQ.md : General/FAQs/What_is_Multiple_Factor_Authentication_MFA.md -Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup_HPC3.md: Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md -General/FAQs/How_to_replace_my_2FA_token.md: General/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md -General/FAQs/How_to_replace_my_2FA.md: General/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md -Scientific_Computing/Terminal_Setup/Ubuntu_LTS_terminal_Windows.md: Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md -General/FAQs/How_can_I_see_how_busy_the_cluster_is.md: General/FAQs/How_busy_is_the_cluster.md -Scientific_Computing/Supported_Applications/Miniconda3.md: Scientific_Computing/Supported_Applications/Miniforge3.md -Scientific_Computing/HPC_Software_Environment/Finding_Software.md : Scientific_Computing/Supported_Applications/index.md hc.md: index.md hc/en-gb.md: index.md Storage/Freezer_long_term_storage.md : Storage/Long_Term_Storage/Freezer_long_term_storage.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Checksums.md: Scientific_Computing/Batch_Jobs/Checksums.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Fair_Share.md: Scientific_Computing/Batch_Jobs/Fair_Share.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Using_GPUs.md: Scientific_Computing/Batch_Jobs/Using_GPUs.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Hyperthreading.md: Scientific_Computing/Batch_Jobs/Hyperthreading.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Job_Checkpointing.md: Scientific_Computing/Batch_Jobs/Job_Checkpointing.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Job_prioritisation.md: Scientific_Computing/Batch_Jobs/Job_prioritisation.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/SLURM-Best_Practice.md: Scientific_Computing/Batch_Jobs/SLURM-Best_Practice.md -Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Slurm_Interactive_Sessions.md: Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md -Scientific_Computing/HPC_Software_Environment/Installing_Third_Party_applications.md : Scientific_Computing/HPC_Software_Environment/Installing_Applications_Yourself.md +Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Job_prioritisation.md: Batch_Computing/Job_prioritisation.md +Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/SLURM-Best_Practice.md: Batch_Computing/SLURM-Best_Practice.md +Scientific_Computing/HPC_Software_Environment/Installing_Third_Party_applications.md : Software/Installing_Applications_Yourself.md +Scientific_Computing/Supported_Applications/ABAQUS.md : Software/Available_Applications/ABAQUS.md +Scientific_Computing/Supported_Applications/AlphaFold.md : Software/Available_Applications/AlphaFold.md +Scientific_Computing/Supported_Applications/ANSYS.md : Software/Available_Applications/ANSYS.md +Scientific_Computing/Supported_Applications/Apptainer.md : Software/Available_Applications/Apptainer.md +Scientific_Computing/Supported_Applications/BLAST.md : Software/Available_Applications/BLAST.md +Scientific_Computing/Supported_Applications/BRAKER.md : Software/Available_Applications/BRAKER.md +Scientific_Computing/Supported_Applications/CESM.md : Software/Available_Applications/CESM.md +Scientific_Computing/Supported_Applications/Clair3.md : Software/Available_Applications/Clair3.md +Scientific_Computing/Supported_Applications/COMSOL.md : Software/Available_Applications/COMSOL.md +Scientific_Computing/Supported_Applications/Cylc.md : Software/Available_Applications/Cylc.md +Scientific_Computing/Supported_Applications/Delft3D.md : Software/Available_Applications/Delft3D.md +Scientific_Computing/Supported_Applications/Dorado.md : Software/Available_Applications/Dorado.md +Scientific_Computing/Supported_Applications/fastStructure.md : Software/Available_Applications/fastStructure.md +Scientific_Computing/Supported_Applications/FDS.md : Software/Available_Applications/FDS.md +Scientific_Computing/Supported_Applications/FlexiBLAS.md : Software/Available_Applications/FlexiBLAS.md +Scientific_Computing/Supported_Applications/FreeSurfer.md : Software/Available_Applications/FreeSurfer.md +Scientific_Computing/Supported_Applications/GATK.md : Software/Available_Applications/GATK.md +Scientific_Computing/Supported_Applications/Gaussian.md : Software/Available_Applications/Gaussian.md +Scientific_Computing/Supported_Applications/GROMACS.md : Software/Available_Applications/GROMACS.md +Scientific_Computing/Supported_Applications/index.md : Software/Available_Applications/index.md +Scientific_Computing/Supported_Applications/ipyrad.md : Software/Available_Applications/ipyrad.md +Scientific_Computing/Supported_Applications/Java.md : Software/Available_Applications/Java.md +Scientific_Computing/Supported_Applications/Julia.md : Software/Available_Applications/Julia.md +Scientific_Computing/Supported_Applications/Keras.md : Software/Available_Applications/Keras.md +Scientific_Computing/Supported_Applications/Lambda_Stack.md : Software/Available_Applications/Lambda_Stack.md +Scientific_Computing/Supported_Applications/MAKER.md : Software/Available_Applications/MAKER.md +Scientific_Computing/Supported_Applications/MATLAB.md : Software/Available_Applications/MATLAB.md +Scientific_Computing/Supported_Applications/Miniforge3.md : Software/Available_Applications/Miniforge3.md +Scientific_Computing/Supported_Applications/Molpro.md : Software/Available_Applications/Molpro.md +Scientific_Computing/Supported_Applications/NWChem.md : Software/Available_Applications/NWChem.md +Scientific_Computing/Supported_Applications/ont-guppy-gpu.md : Software/Available_Applications/ont-guppy-gpu.md +Scientific_Computing/Supported_Applications/OpenFOAM.md : Software/Available_Applications/OpenFOAM.md +Scientific_Computing/Supported_Applications/OpenSees.md : Software/Available_Applications/OpenSees.md +Scientific_Computing/Supported_Applications/ORCA.md : Software/Available_Applications/ORCA.md +Scientific_Computing/Supported_Applications/ParaView.md : Software/Available_Applications/ParaView.md +Scientific_Computing/Supported_Applications/Python.md : Software/Available_Applications/Python.md +Scientific_Computing/Supported_Applications/R.md : Software/Available_Applications/R.md +Scientific_Computing/Supported_Applications/RAxML.md : Software/Available_Applications/RAxML.md +Scientific_Computing/Supported_Applications/Relion.md : Software/Available_Applications/Relion.md +Scientific_Computing/Supported_Applications/snpEff.md : Software/Available_Applications/snpEff.md +Scientific_Computing/Supported_Applications/Supernova.md : Software/Available_Applications/Supernova.md +Scientific_Computing/Supported_Applications/Synda.md : Software/Available_Applications/Synda.md +Scientific_Computing/Supported_Applications/TensorFlow_on_CPUs.md : Software/Available_Applications/TensorFlow_on_CPUs.md +Scientific_Computing/Supported_Applications/TensorFlow_on_GPUs.md : Software/Available_Applications/TensorFlow_on_GPUs.md +Scientific_Computing/Supported_Applications/Trinity.md : Software/Available_Applications/Trinity.md +Scientific_Computing/Supported_Applications/VASP.md : Software/Available_Applications/VASP.md +Scientific_Computing/Supported_Applications/VirSorter.md : Software/Available_Applications/VirSorter.md +Scientific_Computing/Supported_Applications/VTune.md : Software/Available_Applications/VTune.md +Scientific_Computing/Supported_Applications/WRF.md : Software/Available_Applications/WRF.md +Scientific_Computing/Profiling_and_Debugging/Debugging.md : Software/Profiling_and_Debugging/Debugging.md +Scientific_Computing/Profiling_and_Debugging/Profiler-ARM_MAP.md : Software/Profiling_and_Debugging/Profiler-ARM_MAP.md +Scientific_Computing/Profiling_and_Debugging/Profiler-VTune.md : Software/Profiling_and_Debugging/Profiler-VTune.md +Scientific_Computing/Profiling_and_Debugging/Slurm_Native_Profiling.md : Software/Profiling_and_Debugging/Slurm_Native_Profiling.md +Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_on_gpu.md : Software/Containers/Run_an_executable_under_Apptainer_on_gpu.md +Scientific_Computing/HPC_Software_Environment/Run_an_executable_under_Apptainer_in_parallel.md : Software/Containers/Run_an_executable_under_Apptainer_in_parallel.md +Scientific_Computing/HPC_Software_Environment/NVIDIA_GPU_Containers.md : Software/Containers/NVIDIA_GPU_Containers.md +Scientific_Computing/HPC_Software_Environment/Software_Version_Management.md : Software/Software_Version_Management.md +Scientific_Computing/HPC_Software_Environment/Software_Installation_Request.md : Software/Software_Installation_Request.md +Scientific_Computing/HPC_Software_Environment/Installing_Applications_Yourself.md : Software/Installing_Applications_Yourself.md +Scientific_Computing/Batch_Jobs/Checking_resource_usage.md : Batch_Computing/Checking_resource_usage.md +Scientific_Computing/Batch_Jobs/Checksums.md : Batch_Computing/Checksums.md +Scientific_Computing/Batch_Jobs/Fair_Share.md : Batch_Computing/Fair_Share.md +Scientific_Computing/Batch_Jobs/Hardware.md : Batch_Computing/Hardware.md +Scientific_Computing/Batch_Jobs/Job_Checkpointing.md : Batch_Computing/Job_Checkpointing.md +Scientific_Computing/Batch_Jobs/Job_Limits.md : Batch_Computing/Job_Limits.md +Scientific_Computing/Batch_Jobs/Job_prioritisation.md : Batch_Computing/Job_prioritisation.md +Scientific_Computing/Batch_Jobs/SLURM-Best_Practice.md : Batch_Computing/SLURM-Best_Practice.md +Scientific_Computing/HPC_Software_Environment/Temporary_directories.md : Batch_Computing/Temporary_directories.md +Scientific_Computing/Running_Jobs_on_Maui_and_Mahuika/Slurm_Interactive_Sessions.md: Interactive_Computing/Slurm_Interactive_Sessions.md +Scientific_Computing/Batch_Jobs/Slurm_Interactive_Sessions.md : Interactive_Computing/Slurm_Interactive_Sessions.md +Scientific_Computing/Interactive_computing_with_OnDemand/how_to_guide.md : Interactive_Computing/OnDemand/how_to_guide.md +Scientific_Computing/Interactive_computing_with_OnDemand/index.md : Interactive_Computing/OnDemand/index.md +Scientific_Computing/Interactive_computing_with_OnDemand/ood_troubleshooting.md : Interactive_Computing/OnDemand/ood_troubleshooting.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/MATLAB.md : Interactive_Computing/OnDemand/Apps/MATLAB.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/RStudio.md : Interactive_Computing/OnDemand/Apps/RStudio.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/virtual_desktop.md : Interactive_Computing/OnDemand/Apps/virtual_desktop.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/VSCode.md : Interactive_Computing/OnDemand/Apps/VSCode.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/index.md : Interactive_Computing/OnDemand/Apps/JupyterLab/index.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md : Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Manual_management.md +Scientific_Computing/Interactive_computing_with_OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md : Interactive_Computing/OnDemand/Apps/JupyterLab/Jupyter_kernels_Tool_assisted_management.md +Scientific_Computing/Interactive_computing_with_OnDemand/Release_Notes/index.md : Interactive_Computing/OnDemand/Release_Notes/index.md +General/Announcements/Accessing_NeSI_Support_during_the_Easter_break.md : Announcements/Accessing_NeSI_Support_during_the_Easter_break.md +General/Announcements/Autodeletion_returning_for_scratch_filesystem.md : Announcements/Autodeletion_returning_for_scratch_filesystem.md +General/Announcements/December_holiday_support_restrictions.md : Announcements/December_holiday_support_restrictions.md +General/Announcements/Identity_Changes_for_Crown_Research_Institutes.md : Announcements/Identity_Changes_for_Crown_Research_Institutes.md +General/Announcements/Known_Issues_HPC3.md : Announcements/Known_Issues_HPC3.md +General/Release_Notes/index.md : Announcements/Release_Notes/index.md +Scientific_Computing/Terminal_Setup/Git_Bash_Windows.md : Getting_Started/Accessing_the_HPCs/Git_Bash_Windows.md +Scientific_Computing/Terminal_Setup/MobaXterm_Setup_Windows.md : Getting_Started/Accessing_the_HPCs/MobaXterm_Setup_Windows.md +Scientific_Computing/Terminal_Setup/Standard_Terminal_Setup.md : Getting_Started/Accessing_the_HPCs/Standard_Terminal_Setup.md +Scientific_Computing/Terminal_Setup/VSCode.md : Getting_Started/Accessing_the_HPCs/VSCode.md +Scientific_Computing/Terminal_Setup/Windows_Subsystem_for_Linux_WSL.md : Getting_Started/Accessing_the_HPCs/Windows_Subsystem_for_Linux_WSL.md +Scientific_Computing/Terminal_Setup/WinSCP-PuTTY_Setup_Windows.md : Getting_Started/Accessing_the_HPCs/WinSCP-PuTTY_Setup_Windows.md +Scientific_Computing/Terminal_Setup/X11.md : Getting_Started/Accessing_the_HPCs/X11.md +Getting_Started/Next_Steps/Parallel_Execution.md : Software/Parallel_Computing/Parallel_Execution.md +Getting_Started/Next_Steps/Multithreading_Scaling_Example.md : Software/Parallel_Computing/Multithreading_Scaling_Example.md +Getting_Started/Next_Steps/MPI_Scaling_Example.md : Software/Parallel_Computing/MPI_Scaling_Example.md +Getting_Started/Next_Steps/Finding_Job_Efficiency.md : Software/Profiling_and_Debugging/Finding_Job_Efficiency.md +Getting_Started/Next_Steps/Job_Scaling_Ascertaining_job_dimensions.md : Software/Profiling_and_Debugging/Job_Scaling_Ascertaining_job_dimensions.md +Software/OpenMP_settings.md : Software/Parallel_Computing/OpenMP_settings.md +Software/Configuring_Dask_MPI_jobs.md : Software/Parallel_Computing/Configuring_Dask_MPI_jobs.md +General/Policy/Acceptable_Use_Policy.md : Getting_Started/Policy/Acceptable_Use_Policy.md +General/Policy/Access_Policy.md : Getting_Started/Policy/Access_Policy.md +General/Policy/Account_Requests_for_non_Tuakiri_Members.md : Getting_Started/Policy/Account_Requests_for_non_Tuakiri_Members.md +General/Policy/Acknowledgement-Citation_and_Publication.md : Getting_Started/Policy/Acknowledgement-Citation_and_Publication.md +General/Policy/Allocation_classes.md : Getting_Started/Policy/Allocation_classes.md +General/Policy/Application_Support_Model.md : Getting_Started/Policy/Application_Support_Model.md +General/Policy/How_we_review_applications.md : Getting_Started/Policy/How_we_review_applications.md +General/Policy/Institutional_allocations.md : Getting_Started/Policy/Institutional_allocations.md +General/Policy/Licence_Policy.md : Getting_Started/Policy/Licence_Policy.md +General/Policy/Merit_allocations.md : Getting_Started/Policy/Merit_allocations.md +General/Policy/Postgraduate_allocations.md : Getting_Started/Policy/Postgraduate_allocations.md +General/Policy/Privacy_Policy.md : Getting_Started/Policy/Privacy_Policy.md +General/Policy/Proposal_Development_allocations.md : Getting_Started/Policy/Proposal_Development_allocations.md +General/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md : Getting_Started/FAQs/Can_I_change_my_time_zone_to_New_Zealand_time.md +General/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md : Getting_Started/FAQs/Can_I_use_SSHFS_to_mount_the_cluster_filesystem_on_my_local_machine.md +General/FAQs/Common_questions_about_the_platform_refresh.md : Getting_Started/FAQs/Common_questions_about_the_platform_refresh.md +General/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md : Getting_Started/FAQs/Converting_from_Windows_style_to_UNIX_style_line_endings.md +General/FAQs/How_busy_is_the_cluster.md : Getting_Started/FAQs/How_busy_is_the_cluster.md +General/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md : Getting_Started/FAQs/How_can_I_give_read_only_team_members_access_to_my_files.md +General/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md : Getting_Started/FAQs/How_can_I_let_my_fellow_project_team_members_read_or_write_my_files.md +General/FAQs/How_can_I_view_images_generated_on_the_cluster.md : Getting_Started/FAQs/How_can_I_view_images_generated_on_the_cluster.md +General/FAQs/How_do_I_find_out_the_size_of_a_directory.md : Getting_Started/FAQs/How_do_I_find_out_the_size_of_a_directory.md +General/FAQs/How_do_I_fix_my_locale_and_language_settings.md : Getting_Started/FAQs/How_do_I_fix_my_locale_and_language_settings.md +General/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md : Getting_Started/FAQs/How_do_I_replace_my_Additional_Authentication_Credentials.md +General/FAQs/How_do_I_request_memory.md : Getting_Started/FAQs/How_do_I_request_memory.md +General/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md : Getting_Started/FAQs/How_do_I_run_my_Python_Notebook_through_SLURM.md +General/FAQs/Ive_run_out_of_storage_space.md : Getting_Started/FAQs/Ive_run_out_of_storage_space.md +General/FAQs/Login_Troubleshooting.md : Getting_Started/FAQs/Login_Troubleshooting.md +General/FAQs/Mahuika_HPC3_Differences.md : Getting_Started/FAQs/Mahuika_HPC3_Differences.md +General/FAQs/What_are_my-bashrc_and-bash_profile_for.md : Getting_Started/FAQs/What_are_my-bashrc_and-bash_profile_for.md +General/FAQs/What_does_oom_kill_mean.md : Getting_Started/FAQs/What_does_oom_kill_mean.md +General/FAQs/What_is_a_core_file.md : Getting_Started/FAQs/What_is_a_core_file.md +General/FAQs/What_Is_A_Trusted_Device.md : Getting_Started/FAQs/What_Is_A_Trusted_Device.md +General/FAQs/What_is_Multiple_Factor_Authentication_MFA.md : Getting_Started/FAQs/What_is_Multiple_Factor_Authentication_MFA.md +General/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md : Getting_Started/FAQs/What_software_environments_are_optimised_for_Machine_Learning_and_data_science.md +General/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md : Getting_Started/FAQs/Where_should_I_store_my_data_on_NeSI_systems.md +General/FAQs/Why_am_I_seeing_Account_is_not_ready.md : Getting_Started/FAQs/Why_am_I_seeing_Account_is_not_ready.md +General/FAQs/Why_does_my_program_crash.md : Getting_Started/FAQs/Why_does_my_program_crash.md +General/FAQs/Why_is_my_job_taking_a_long_time_to_start.md : Getting_Started/FAQs/Why_is_my_job_taking_a_long_time_to_start.md +Getting_Started/Next_Steps/Submitting_your_first_job.md : Batch_Computing/Submitting_your_first_job.md +Getting_Started/Next_Steps/Moving_files_to_and_from_the_cluster.md : Storage/Moving_files_to_and_from_the_cluster.md +Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md : Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC_YouTube_Recordings.md +Scientific_Computing/Training/Introduction_to_computing_on_the_NeSI_HPC.md : Getting_Started/Getting_Help/Introduction_to_computing_on_the_NeSI_HPC.md +Scientific_Computing/Training/Webinars.md : Getting_Started/Getting_Help/Webinars.md +Scientific_Computing/Training/Workshops.md : Getting_Started/Getting_Help/Workshops.md +Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_NeSI_project.md: Getting_Started/Projects/Applying_for_a_New_Project.md +General/NeSI_Policies/Proposal_Development_allocations.md : Getting_Started/Policy/Proposal_Development_allocations.md +Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_an_existing_NeSI_project.md: Getting_Started/Projects/Applying_to_Join_a_Project.md +General/NeSI_Policies/Account_Requests_for_non_Tuakiri_Members.md: Getting_Started/Policy/Account_Requests_for_non_Tuakiri_Members.md +General/Announcements/HPC3.md: Getting_Started/Creating_an_Account.md +Service_Subscriptions/Contracts_and_billing_processes/Billing_process.md : Service_Subscriptions/Contracts_and_Billing/Billing_process.md +Service_Subscriptions/Contracts_and_billing_processes/Types_of_contracts.md : Service_Subscriptions/Contracts_and_Billing/Types_of_contracts.md +Getting_Started/Accounts-Projects_and_Allocations/Creating_an_Account_Profile.md : Getting_Started/Creating_an_Account.md +Getting_Started/Accounts-Projects_and_Allocations/Adding_members_to_your_project.md : Getting_Started/Projects/Adding_members_to_your_Project.md +Getting_Started/Accounts-Projects_and_Allocations/Applying_for_a_new_project.md : Getting_Started/Projects/Applying_for_a_New_Project.md +Getting_Started/Accounts-Projects_and_Allocations/Applying_to_join_a_project.md : Getting_Started/Projects/Applying_to_Join_a_Project.md +Getting_Started/Projects/Adding_members_to_your_project.md : Getting_Started/Projects/Adding_Members_to_your_Project.md +Getting_Started/Projects/Applying_to_join_a_project.md : Getting_Started/Projects/Applying_to_Join_a_Project.md +Getting_Started/Projects/Applying_for_a_new_project.md : Getting_Started/Projects/Applying_for_a_New_Project.md +Getting_Started/Accounts-Projects_and_Allocations/Allocations_and_Extensions.md : Getting_Started/Allocations/Allocations_and_Extensions.md +Getting_Started/Accounts-Projects_and_Allocations/Quarterly_allocation_periods.md : Getting_Started/Allocations/Quarterly_allocation_periods.md +Getting_Started/Accounts-Projects_and_Allocations/What_is_an_allocation.md : Getting_Started/Allocations/What_is_an_allocation.md +Getting_Started/Creating_an_Account_Profile.md : Getting_Started/Creating_an_Account.md diff --git a/fixlinks.py b/fixlinks.py new file mode 100644 index 000000000..685a351f6 --- /dev/null +++ b/fixlinks.py @@ -0,0 +1,87 @@ +# Tries to replace all internal broken links with less broken ones. +# Note: Partially AI generated. Not to be trusted. +import argparse, os, re, sys +from pathlib import Path + +MD_ROOT = Path("docs") +LINK_RE = re.compile(r'\[([^\]]+)\]\(([^)]+)\)') + +def all_md_files(root): + return [p for p in root.rglob("**/*.md")] + +def resolve_path(base_md, target): + # separate anchor + target_path, *anchor = target.split('#',1) + anchor = ('#' + anchor[0]) if anchor else '' + if not target_path: + return False # anchor-only + # if target is directory index usually index.md? + cand = (base_md.parent / target_path).resolve() + # try direct existence + if cand.exists(): + return True + return False + +def find_exact_candidates(basename, root): + return [p for p in root.rglob("*.md") if p.name == basename] + +def find_similar_candidates(basename, root): + return [p for p in root.rglob("*.md") if jaccard_similarity(p.name, basename) > 0.5] + + +def jaccard_similarity(s1, s2): + set1 = set(s1.split(".")[0].lower().split("_")) # Split into words + set2 = set(s2.split(".")[0].lower().split("_")) + intersection = len(set1.intersection(set2)) + union = len(set1.union(set2)) + if (intersection / union) > 0: + print(f"{s2}:{s1} {intersection / union}") + return intersection / union + +def main(dry_run): + md_files = all_md_files(MD_ROOT) + fixes = [] + for md in md_files: + text = md.read_text(encoding="utf8") + changed = text + for m in LINK_RE.finditer(text): + target = m.group(2).strip() + if target.startswith(("http://","https://","mailto:","/")): + continue + if resolve_path(md, target): + continue + # not found: try to find file by basename + base = os.path.basename(target.split('#',1)[0]) + if not base: + continue + candidates = find_exact_candidates(base, MD_ROOT) + if ~len(candidates): + candidates = find_similar_candidates(base, MD_ROOT) + if len(candidates) == 1: + rel = os.path.relpath(candidates[0], md.parent) + # preserve anchor + anchor = '' + if '#' in target: + anchor = '#' + target.split('#',1)[1] + new_target = rel.replace(os.path.sep, "/") + anchor + fixes.append((md, target, new_target)) + changed = changed.replace("(%s)" % target, "(%s)" % new_target) + elif len(candidates) > 1: + print("MULTIPLE CANDIDATES:", md, target, "=>", [str(p) for p in candidates]) + else: + print("NO CANDIDATE:", md, target) + if fixes and not dry_run: + md.write_text(changed, encoding="utf8") + # report + if fixes: + print("\nProposed / Applied fixes:") + for f in fixes: + print(f[0], ":", f[1], "=>", f[2]) + else: + print("No fixes proposed.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="will not write out unless present") + args = parser.parse_args() + main(dry_run=not args.apply) diff --git a/requirements.in b/requirements.in index c44594ca3..38b04a3cb 100644 --- a/requirements.in +++ b/requirements.in @@ -22,6 +22,7 @@ linkcheckmd symspellpy pyspelling flashtext +titlecase # additional tools pip-tools diff --git a/requirements.txt b/requirements.txt index 1a28ddf51..777f07143 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ aiosignal==1.4.0 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.11.0 +anyio==4.12.0 # via httpx attrs==25.4.0 # via aiohttp @@ -22,7 +22,7 @@ babel==2.17.0 # mkdocs-material backrefs==6.1 # via mkdocs-material -beautifulsoup4==4.14.2 +beautifulsoup4==4.14.3 # via pyspelling bracex==2.6 # via wcmatch @@ -56,9 +56,9 @@ cryptography==46.0.3 # via pyjwt editdistpy==0.1.6 # via symspellpy -essentials==1.1.8 +essentials==1.1.9 # via essentials-openapi -essentials-openapi==1.2.1 +essentials-openapi==1.3.0 # via neoteroi-mkdocs filelock==3.20.0 # via cachecontrol @@ -146,7 +146,7 @@ mkdocs==1.6.1 # mkdocs-simple-hooks # mkdocs-spellcheck # neoteroi-mkdocs -mkdocs-awesome-nav==3.2.0 +mkdocs-awesome-nav==3.3.0 # via -r requirements.in mkdocs-bootstrap4==0.1.5 # via -r requirements.in @@ -174,7 +174,7 @@ mkdocs-section-index==0.3.10 # via -r requirements.in mkdocs-simple-hooks==0.1.5 # via -r requirements.in -mkdocs-spellcheck==1.1.2 +mkdocs-spellcheck==1.2.0 # via -r requirements.in msgpack==1.1.2 # via cachecontrol @@ -184,7 +184,7 @@ multidict==6.7.0 # yarl natsort==8.4.0 # via mkdocs-awesome-nav -neoteroi-mkdocs==1.1.3 +neoteroi-mkdocs==1.2.0 # via -r requirements.in packaging==25.0 # via @@ -209,7 +209,7 @@ proselint==0.16.0 # via -r requirements.in pycparser==2.23 # via cffi -pydantic==2.12.4 +pydantic==2.12.5 # via mkdocs-awesome-nav pydantic-core==2.41.5 # via pydantic @@ -221,7 +221,7 @@ pygments==2.19.2 # rich pyjwt[crypto]==2.10.1 # via pygithub -pymdown-extensions==10.17.1 +pymdown-extensions==10.17.2 # via mkdocs-material pynacl==1.6.1 # via pygithub @@ -229,7 +229,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pyspelling==2.12 +pyspelling==2.12.1 # via -r requirements.in python-dateutil==2.9.0.post0 # via @@ -262,8 +262,6 @@ six==1.17.0 # python-dateutil smmap==5.0.2 # via gitdb -sniffio==1.3.1 - # via anyio soupsieve==2.8 # via # beautifulsoup4 @@ -274,6 +272,8 @@ symspellpy==6.9.0 # via -r requirements.in termcolor==3.2.0 # via mkdocs-macros-plugin +titlecase==2.4.1 + # via -r requirements.in typing-extensions==4.15.0 # via # aiosignal