Implement null if support as a WITH option.

This gives a default "null if" option to all the input columns at once, and
it's still possible to override the default per column.

In passing, fix project-fields declarations that SBCL now complains about
when they're not true, such as declaring a vector when we might have :null
or nil. As a result, remove the (declare (optimize speed)) in the generated
field processing code.

src/parsers/command-csv.lisp

@@ -134,7 +134,8 @@
                         option-fields-terminated-by
                         option-trim-unquoted-blanks
                         option-keep-unquoted-blanks
-                        option-csv-escape-mode))
+                        option-csv-escape-mode
+                        option-null-if))
 
 (defrule csv-options (and kw-with
                              (and csv-option (* (and comma csv-option))))
@@ -429,26 +430,35 @@
        (progn
          ,(sql-code-block pg-db-conn :pre before "before load")
 
-         (let ((on-error-stop             (getf ',options :on-error-stop))
-               (truncate                  (getf ',options :truncate))
-               (disable-triggers          (getf ',options :disable-triggers))
-               (drop-indexes              (getf ',options :drop-indexes))
-               (max-parallel-create-index (getf ',options :max-parallel-create-index))
-               (source
-                (make-instance 'copy-csv
-                               :target-db  ,pg-db-conn
-                               :source     source-db
-                               :target     (create-table ',target-table-name)
-                               :encoding   ,encoding
-                               :fields    ',fields
-                               :columns   ',columns
-                               ,@(remove-batch-control-option
-                                  options :extras '(:worker-count
-                                                    :concurrency
-                                                    :truncate
-                                                    :drop-indexes
-                                                    :disable-triggers
-                                                    :max-parallel-create-index)))))
+         (let* ((on-error-stop             (getf ',options :on-error-stop))
+                (truncate                  (getf ',options :truncate))
+                (disable-triggers          (getf ',options :disable-triggers))
+                (drop-indexes              (getf ',options :drop-indexes))
+                (max-parallel-create-index (getf ',options :max-parallel-create-index))
+                (fields
+                 ',(let ((null-as (getf options :null-as)))
+                     (if null-as
+                         (mapcar (lambda (field)
+                                   (if (member :null-as field) field
+                                       (append field (list :null-as null-as))))
+                                 fields)
+                         fields)))
+                (source
+                 (make-instance 'copy-csv
+                                :target-db  ,pg-db-conn
+                                :source     source-db
+                                :target     (create-table ',target-table-name)
+                                :encoding   ,encoding
+                                :fields    fields
+                                :columns   ',columns
+                                ,@(remove-batch-control-option
+                                   options :extras '(:null-as
+                                                     :worker-count
+                                                     :concurrency
+                                                     :truncate
+                                                     :drop-indexes
+                                                     :disable-triggers
+                                                     :max-parallel-create-index)))))
            (copy-database source
                           ,@ (when worker-count
                                (list :worker-count worker-count))

src/sources/common/project-fields.lisp

@@ -115,12 +115,11 @@
                                           sexp))
                               (t      sexp)))))
 		 `(lambda (row)
-		    (declare (optimize speed) (type list row))
+		    (declare (type list row))
 		    (destructuring-bind (&optional ,@args &rest extra) row
 		      (declare (ignorable ,@args) (ignore extra))
                       (let ,values
-                        (declare (ignorable ,@args)
-                                 (type vector ,@args))
+                        (declare (ignorable ,@args))
                         (vector ,@newrow)))))))))
       ;; allow for some debugging
       (if compile (compile nil projection) projection))))

test/csv-null-if.load

@@ -0,0 +1,22 @@
+LOAD CSV
+     FROM INLINE (id, number, data)
+     INTO postgresql:///pgloader?nullif
+
+     BEFORE LOAD DO
+      $$ drop table if exists nullif; $$,
+      $$ CREATE TABLE nullif
+         (
+            id     serial primary key,
+            number integer,
+            data   text
+         );
+      $$
+
+     WITH null if '\N',
+          fields terminated by ',',
+          fields enclosed by '"',
+          fields escaped by backslash-quote;
+
+
+"1",\N,"testing nulls"
+"2","2","another test"