nextflow.config

#!/usr/bin/env nextflow

// BEAMFORMING OPTIONS
// ----------------------------------------------------------------------------
params.obsid              = "no_obsid" // MWA observation ID (GPS)
params.calid              = "no_calid" // MWA calibration observation ID (GPS)
params.pointings          = null       // Space seperated pointings in the format HH:MM:SS_DD:MM:SS
params.pointing_file      = null       // Path to a file containing a pointing on each line in the format HH:MM:SS_DD:MM:SS
params.bestprof_pointings = null       // Directory of bestprof files from which pointings will be extracted

// Observation length options
params.begin = 0     // Beginning of the observation (GPS)
params.end   = 0     // End of the observation (GPS)
params.all   = false // Use the whole obervation (GPS)

// Beamforming type options
params.summed = false  // Sum the Stoke paramters
params.incoh  = false // Also produce an incoherent beam
params.ipfb   = false // Also produce a high time resolution Inverse Polyphase Filter Bank beam
params.offringa = false // Use offringa calibration solution instead of RTS
// How much temp memory (MB) needed per second per file
if ( params.summed ) {
    params.temp_mem_sec = 1.25
}
else {
    params.temp_mem_sec = 5
}


// PULSAR SEARCH OPTIONS
// ----------------------------------------------------------------------------
params.cand = "Blind" // Label given to output files
params.sp   = false   // Perform only a single pulse search
params.publish_all_prepfold = false // Output all prepfold files (redundant if using the calssifier module)

// Dispersion Measure options
params.dm_min            = 1    // Minimum DM
params.dm_max            = 250  // Maximum DM
params.dm_min_step       = 0.02 // Minimum DM step (lowering increases sensitivity)
params.dm_max_step       = 0.5  // Maximum DM step (lowering increases sensitivity)
params.max_dms_per_job   = 5000 // Maximum number of DM steps per job. Decrease to make smaller jobs
params.max_work_function = 300  // Maximum total work function per job. Decrease to make smaller jobs
params.max_folds_per_job = 5    // Maximum number prepfolds per job. Decrease to make smaller jobs

// Defaults for the accelsearch command
params.nharm      = 16    // Number of harmonics to search
params.min_period = 0.001 // Min period to search for in sec (ANTF min = 0.0013)
params.max_period = 30    // Max period to search for in sec (ANTF max = 23.5)
params.zmax       = 0     // Maximum acceleration to search (0 will do a simpler periodic search)

// This is a module load command that only OzSTAR needs
params.presto_python_load = ""


// FOLLOW UP OPTIONS
// ----------------------------------------------------------------------------
params.bins   = 128     // Number of profile bins
params.subint = 60      // Number of time subintergrations
params.nchan  = 48      // Number of frequency subchannels
params.pulsar = ""      // Pulsar name
params.period = 0.90004 // Period in seconds
params.dm     = 23.123  // Dispersion measure


// SOFTWARE VERSION OPTIONS
// ----------------------------------------------------------------------------
params.vcstools_version   = 'master'
params.mwa_search_version = 'master'
params.presto_version     = 'v4.0_7ec3c83'


// OUTPUT VISUALISATION OPTIONS
// ----------------------------------------------------------------------------
params.output_vis = false
if ( params.output_vis ) {
    dag {
        enabled   = true
        overwrite = true
        file      = "${params.output_vis}_dag.png"
    }
    report {
        enabled   = true
        overwrite = true
        file      = "${params.output_vis}_report.html"
    }
    timeline {
        enabled   = true
        overwrite = true
        file      = "${params.output_vis}_timeline.html"
    }
    trace {
        enabled   = true
        overwrite = true
        file      = "${params.output_vis}_trace.txt"
    }
}


// Memory and duration calculation fucntions
// ----------------------------------------------------------------------------
def vcsbeam_temp_mem(dur, npoints) {
    // Return memory on SSD/NVME needed fo the VCSbeam files in MB
    // dur is in seconds
    // npoints is number of pointins which is set as 2.5 for the IPFB to allow enough room
    return (int) ( params.temp_mem_sec * Float.valueOf(dur) * npoints + 2000 )
}
def presto_temp_mem(dur, ndms_job) {
    // Return memory on SSD/NVME needed fo the presto search files in MB
    // dur is in seconds
    // ndms_job is the number of dedispersion jobs
    return (int) ( 0.12 * Float.valueOf(dur) * Float.valueOf(ndms_job) )
}


// CLUSTER SPECFIC DEFAULTS
// ----------------------------------------------------------------------------
def hostname = "hostname".execute().text.trim().replace("-", "")
if ( hostname.startsWith("garrawarla") ) {
    // Set up for Pawsey's Garrawarla cluster

    // Set up containers
    process.module = 'singularity/3.7.4'
    singularity {
        enabled = true
        runOptions = '--nv -B /nvmetmp'
        envWhitelist = 'SINGULARITY_BINDPATH, SINGULARITYENV_LD_LIBRARY_PATH'
    }
    params.containerDir = '/pawsey/mwa/singularity'

    // Default directories
    params.module_dir        = '/pawsey/mwa/software/python3/modulefiles'
    params.presto_module_dir = '/pawsey/mwa/software/python3/modulefiles'
    params.presto_module     = 'master'
    params.vcsdir            = "/astro/mwavcs/${USER}"
    params.search_dir        = "/astro/mwavcs/${USER}/pulsar_search"
    workDir                  = "/astro/mwavcs/${USER}/${params.obsid}_work"

    process {
        cache = 'lenient'
        // Resource set up
        withLabel: 'gpu|cpu|download' {
            cpus = 1
            executor = 'slurm'
            memory = "10 GB"
        }
        withLabel: gpu {
            queue = 'gpuq'
        }
        withLabel: cpu {
            queue = 'workq'
        }
        withLabel: large_mem {
            memory = "16 GB"
        }
        withLabel: download {
            queue = 'copyq'
            executor = 'slurm-zeus'
        }
        // Software dependency set up
        withLabel: vcsbeam {
            if ( params.ipfb ) {
                clusterOptions = { "--gres=gpu:1  --tmp=${ vcsbeam_temp_mem(dur, 2.5) }MB" }
            }
            else {
                clusterOptions = { "--gres=gpu:1  --tmp=${ vcsbeam_temp_mem(dur, points.size()) }MB" }
            }
            scratch        = '/nvmetmp'
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: vcstools {
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: presto_search {
            container      = "file:///${params.containerDir}/presto/presto_${params.presto_version}.sif"
            clusterOptions = { "--export=NONE --tmp=${ presto_temp_mem(dur, ndms_job) }MB" }
            scratch        = '/nvmetmp'
        }
        withLabel: presto {
            container      = "file:///${params.containerDir}/presto/presto_${params.presto_version}.sif"
        }
        withLabel: sps {
            container      = "file:///${params.containerDir}/sps/sps.sif"
        }
        withLabel: lofar_ml {
            container      = "file:///${params.containerDir}/lofar_pulsar_ml/lofar_pulsar_ml.sif"
        }
        withLabel: lofar_feature_lab {
            container      = "file:///${params.containerDir}/lofar_pulsar_ml/lofar_pulsar_ml.sif"
        }
    }
    executor.submitRateLimit = '100 sec'

    // Beamforming benchmarks
    //params.bm_read  =  0.172 //this read benchmark is not always sufficent
    //params.bm_read  =  3.000 //overkill
    params.bm_read  =  1.000
    params.bm_cal   =  0.091
    params.bm_beam  =  0.033
    params.bm_write =  0.390

    // PRESTO scaling benchmarks (will be adjusted based on observation duration and number of DM steps)
    params.search_scale   = 5.0
    params.prepfold_scale = 16.0

    // Job number controls
    executor.$slurm.queueSize = 1000
    params.max_pointings      = 20
    params.max_gpu_jobs       = 48
    params.max_search_jobs    = 400
    params.max_cpus_per_node  = 38 // leaves 2 for the gpuq
}
else if ( hostname.startsWith("farnarkle") ) {
    // Set up for Swinburnes's Ozstar cluster

    // Set up containers
    singularity {
        enabled = true
        // runOptions = '-B /fred --nv'
        runOptions = '-B /fred'
    }
    params.containerDir = '/fred/oz125/container_images'

    // Default directories
    params.module_dir        = '/fred/oz125/software/modulefiles'
    params.presto_module_dir = '/apps/users/pulsar/skylake/modulefiles'
    params.presto_module     = 'd6265c2'
    params.vcsdir            = '/fred/oz125/vcs'
    params.search_dir        = "/fred/oz125/${USER}/pulsar_search"
    workDir                  = "/fred/oz125/${USER}/${params.obsid}_work"

    process {
        module = 'apptainer/latest'
        cache = 'lenient'
        // Resource set up
        withLabel: 'gpu|cpu' {
            executor = 'slurm'
            cpus     = 1
        }
        withLabel: gpu {
            queue  = 'skylake-gpu'
            memory = "25 GB"
        }
        withLabel: cpu {
            queue  = 'skylake'
            memory = "3 GB"
        }
        withLabel: large_mem {
            memory = "8 GB"
        }
        // Software dependency set up
        withLabel: vcsbeam {
            if ( params.ipfb ) {
                clusterOptions = { "--gres=gpu:1  --tmp=${ vcsbeam_temp_mem(dur, 2.5) }MB" }
            }
            else {
                clusterOptions = { "--gres=gpu:1  --tmp=${ vcsbeam_temp_mem(dur, points.size()) }MB" }
            }
            scratch        = '$JOBFS'
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: vcstools {
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: presto_search {
            beforeScript   = "module use ${params.module_dir}; module load presto/min_path"
            clusterOptions = { "--export=NONE --tmp=${ presto_temp_mem(dur, ndms_job) }MB" }
            scratch        = '$JOBFS'
        }
        withLabel: presto {
            container  = "file:///${params.containerDir}/presto/presto_${params.presto_version}"
            // tmp storage for container image
            clusterOptions = { "--tmp=2000MB" }
            scratch        = '$JOBFS'
        }
        withLabel: sps {
            container      = "file:///${params.containerDir}/sps/sps_sandbox"
            // tmp storage for container image
            clusterOptions = { "--tmp=2000MB" }
            scratch        = '$JOBFS'
        }
        withLabel: lofar_ml {
            beforeScript   = "module use ${params.module_dir}; module load LOTAASClassifier/master"
        }
        withLabel: lofar_feature_lab {
            beforeScript   = "module use ${params.module_dir}; module load PulsarFeatureLab/V1.3.2"
        }
    }
    executor.submitRateLimit = '100 sec'
    // This is a module load command that only OzSTAR needs
    // params.presto_python_load = "module use ${params.presto_module_dir}; module load presto/${params.presto_module}; module load python/2.7.14; module load matplotlib/2.2.2-python-2.7.14"
    params.presto_python_load = "module use ${params.presto_module_dir}; module load presto/${params.presto_module}"//"; module load python/2.7.14; module load matplotlib/2.2.2-python-2.7.14"

    // Beamforming benchmarks
    params.bm_read  = 1.200 // 0.366
    params.bm_cal   = 0.115
    params.bm_beam  = 0.041
    params.bm_write = 0.071

    // PRESTO scaling benchmarks (will be adjusted based on observation duration and number of DM steps)
    params.search_scale   = 5.0
    params.prepfold_scale = 24.0

    // Job number controls
    executor.$slurm.queueSize = 1000
    params.max_pointings      = 120
    params.max_gpu_jobs       = 120
    params.max_search_jobs    = 800
}
else if ( hostname.startsWith("x86") ) {
    // Set up for the China SKA Region Centre Prototype's ARM cluster

    process {
        cache = 'lenient'
        // Resource set up
        withLabel: 'gpu|cpu|cpu_large_mem' {
            executor = 'slurm'
            cpus     = 1
        }
        withLabel: gpu {
            queue  = 'all-gpu'
            memory = "30 GB"
        }
        withLabel: cpu {
            queue  = 'purley-cpu'
            memory = "3 GB"
        }
        withLabel: large_mem {
            memory = "8 GB"
        }
        // Software dependency set up
        withLabel: vcsbeam {
            clusterOptions = "--gres=gpu:1"
            scratch        = '/ssd'
            //container = "file:///${params.containerDir}/vcstools/vcstools_${params.vcstools_version}.sif"
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: vcstools {
            beforeScript   = "module use ${params.module_dir}; module load vcstools/${params.vcstools_version}"
        }
        withLabel: presto_search {
            container      = "file:///${params.containerDir}/presto/presto.sif"
            clusterOptions = { "--export=NONE --tmp=${ presto_temp_mem(dur, ndms_job) }MB" }
            scratch        = '/ssd'
        }
        withLabel: presto {
            container      = "file:///${params.containerDir}/presto/presto.sif"
        }
        withLabel: sps {
            container      = "nickswainston/sps"
        }
        withLabel: lofar_ml {
            container      = "file:///${params.containerDir}/lofar_pulsar_ml/lofar_pulsar_ml.sif"
        }
        withLabel: lofar_feature_lab {
            container      = "file:///${params.containerDir}/lofar_pulsar_ml/lofar_pulsar_ml.sif"
        }
    }
    executor.submitRateLimit = '100 sec'

    // Set up containers
    singularity {
        enabled = true
        runOptions = '-B /o9000'
    }
    params.containerDir = '/o9000/MWA/Pulsar/vcs/singularity_images'

    // Default directories
    params.module_dir        = '/home/app/modulefiles/'
    params.presto_module_dir = '/home/app/modulefiles/'
    params.presto_module     = 'cpu-master'
    params.vcsdir            = '/o9000/MWA/Pulsar/vcs'
    params.search_dir        = '/o9000/MWA/Pulsar/vcs/candidates'
    workDir                  = "/o9000/MWA/Pulsar/vcs/${params.obsid}/${env.USER}_work"

    // Beamforming benchmarks
    //params.bm_read  = 0.266 previous benchmark
    params.bm_read  = 0.366*2
    params.bm_cal   = 0.070*2
    params.bm_beam  = 0.120*2
    params.bm_write = 0.013*2

    // PRESTO scaling benchmarks (will be adjusted based on observation duration and number of DM steps)
    params.search_scale   = 10.0
    params.prepfold_scale = 50

    // Job number controls
    executor.$slurm.queueSize = 1000
    params.max_pointings      = 120
    params.max_gpu_jobs       = 120
    params.max_search_jobs    = 800
}
else {
    // No recognised hostname so assuming defaults

    // Resource set up
    executor.name = 'local'
    executor.queueSize = 8

    // Software dependency set up
    process {
        withLabel: vcsbeam {
            container = "cirapulsarsandtransients/vcstools:${params.vcstools_version}"
        }
        withLabel: vcstools {
            container = "cirapulsarsandtransients/vcstools:${params.vcstools_version}"
        }
        withLabel: 'presto_search|presto' {
            container = "nickswainston/presto:${params.presto_version}"
        }
        withLabel: lofar_ml {
            container = "nickswainston/lofar_pulsar_ml"
        }
        withLabel: lofar_feature_lab {
            container = "cirapulsarsandtransients/pulsarfeaturelab:V1.3.2"
        }
    }

    // Set up containers
    docker.enabled = true

    // Default directories
    params.vcsdir     = './'
    params.search_dir = './'

    // Beamforming benchmarks
    params.bm_read  = 0.366*2
    params.bm_cal   = 0.070*2
    params.bm_beam  = 0.120*2
    params.bm_write = 0.013*2

    // PRESTO scaling benchmarks (will be adjusted based on observation duration and number of DM steps)
    params.search_scale   = 10.0
    params.prepfold_scale = 50

    // Job number controls
    params.max_pointings   = 10
    params.max_gpu_jobs    = 1
    params.max_search_jobs = 16
}


// File and directory optins
params.no_combined_check = false // Don't perform a check of the VCS files
params.publish_fits      = false // Publish the fits files to the sub directories
params.publish_all_classifer_cands = true
params.out_dir = "${params.search_dir}/${params.obsid}_candidates"
// Default directory of calibration solutions
if ( params.offringa ) {
    params.didir = "${params.vcsdir}/${params.obsid}/cal/${params.calid}/offringa"
}
else {
    params.didir = "${params.vcsdir}/${params.obsid}/cal/${params.calid}/rts"
}