ENH Earlier checks for permissions

For example, if a single file is going to cause an error, paired files
were often processed first before an error was triggered. Now, an error
is triggered in seconds

ChangeLog

@@ -1,6 +1,7 @@
 Unreleased
 	* Fix bug when outputing comments to compressed files
 	* Better performance for large uses of the parallel module
+	* Faster file permission checks for readsets
 
 Version 1.5.0 2022-09-14 by luispedro
 	* Add `compress_level` option to `write()` function

NGLess/BuiltinModules/Checks.hs

@@ -1,8 +1,7 @@
-{- Copyright 2016-2019 NGLess Authors
+{- Copyright 2016-2022 NGLess Authors
  - License: MIT
  -}
 
-{-# LANGUAGE TupleSections, OverloadedStrings #-}
 
 module BuiltinModules.Checks
     ( checkOFile
@@ -20,7 +19,9 @@ import System.FilePath (takeDirectory)
 import Language
 
 import Modules
+import Output
 import NGLess
+import Data.FastQ (FastQFilePath(..))
 import           Utils.Suggestion (checkFileReadable)
 
 checkOFile :: T.Text -> IO (Maybe T.Text)
@@ -62,6 +63,18 @@ executeChecks "__check_index_access" (NGOList vs) args = do
                         else "")
                     ])
     return NGOVoid
+
+executeChecks "__check_readset" (NGOReadSet name (ReadSet pairs singles)) args = do
+    lno <- lookupIntegerOrScriptError "readset check" "original_lno" args
+    let catPairs [] = []
+        catPairs ((a,b):xs) = a:b:catPairs xs
+    forM_ (catPairs pairs ++ singles) $ \(FastQFilePath _ r) -> do
+        outputListLno TraceOutput (Just $ fromEnum lno) ["Checking file ", r, "."]
+        merr <- liftIO (checkFileReadable r)
+        whenJust merr $ \err ->
+            throwSystemError $! concat ["Cannot read file '", r, "' for sample '", T.unpack name, "'. ",
+                                T.unpack err, " (used in line ", show lno, ")."]
+    return NGOVoid
 executeChecks _ _ _ = throwShouldNotOccur "checks called in an unexpected fashion."
 
 indexCheck = Function
@@ -95,11 +108,20 @@ iFileCheck = Function
     , funcAllowsAutoComprehension = False
     , funcChecks = []
     }
+iRSCheck = Function
+    { funcName = FuncName "__check_readset"
+    , funcArgType = Just NGLReadSet
+    , funcArgChecks = []
+    , funcRetType = NGLVoid
+    , funcKwArgs = [ArgInformation "original_lno" True NGLInteger []]
+    , funcAllowsAutoComprehension = False
+    , funcChecks = []
+    }
 
 loadModule :: T.Text -> NGLessIO Module
 loadModule _ = return def
     { modInfo = ModInfo "builtin.checks" "0.0"
-    , modFunctions = [oFileCheck, iFileCheck, indexCheck]
+    , modFunctions = [oFileCheck, iFileCheck, indexCheck, iRSCheck]
     , runFunction = executeChecks
     }
 

NGLess/Transform.hs

@@ -76,6 +76,7 @@ transform mods sc = Script (nglHeader sc) <$> applyM transforms (nglBody sc)
                 , ifLenDiscardSpecial
                 , substrimReassign
                 , addFileChecks
+                , addRSChecks
                 , addIndexChecks
                 , addUseNewer
                 , addCountsCheck
@@ -354,6 +355,20 @@ addFileChecks' checkFname tag ((lno,e):rest) = do
         validVariables (ConstStr _) = []
         validVariables _ = [Variable "this", Variable "wont", Variable "work"] -- this causes the caller to bailout
 
+
+addRSChecks :: [(Int,Expression)] -> NGLessIO [(Int, Expression)]
+addRSChecks = return . genericCheckUpfloat addRSChecks'
+
+addRSChecks' :: (Int, Expression) -> Maybe ([Variable],Expression)
+addRSChecks' (lno, e) = case e of
+    Assignment _ (FunctionCall (FuncName "preprocess") lk@(Lookup _ v) _ _)
+        -> Just ([v],
+                FunctionCall (FuncName "__check_readset")
+                            lk
+                            [(Variable "original_lno", ConstInt (toInteger lno))]
+                            Nothing)
+    _ -> Nothing
+
 -- | 'addIndexChecks' implements the following transformation
 --
 -- array = <non constant expression>

tests/error-check-file-early/list.yaml

@@ -0,0 +1,5 @@
+samples:
+  sample:
+    - paired:
+        - sample/funny_one.gz
+        - sample/funny_two.gz

tests/error-check-file-early/load_sample_list.ngl

@@ -0,0 +1,14 @@
+ngless "1.5"
+
+samples = load_sample_list('list.yaml')
+
+input = samples[0]
+
+println(input.name())
+
+trim = 3
+input = preprocess(input) using |read|:
+   read = read[trim:]
+   if len(read) < 10:
+       discard
+write(input, ofile='output.fq')

tests/error-check-file-early/sample/funny_one.gz

@@ -0,0 +1 @@
+../../load_sample_list/sample/funny_one.gz