Improve CSV schema validation with filename patterns

This commit introduces filename pattern validation to the CSV schema, which allows increased data consistency checks. It also extends the capability to include additional columns to the schema, providing a more flexible structure. Enhancements were also made to error handling, introducing a quick-stop feature to expedite error discovery.

README.md

@@ -307,6 +307,11 @@ This gives you great flexibility when validating CSV files.
 ```yml
 # It's a full example of the CSV schema file in YAML format.
 
+# Regular expression to match the file name. If not set, then no pattern check
+# This way you can validate the file name before the validation process.
+# Feel free to check parent directories as well.
+filename_pattern: /demo(-\d+)?\.csv$/
+
 csv: # Here are default values. You can skip this section if you don't need to override the default values
   header: true                          # If the first row is a header. If true, name of each column is required
   delimiter: ,                          # Delimiter character in CSV file
@@ -362,6 +367,8 @@ columns:
       cardinal_direction: true          # Valid cardinal direction. Examples: "N", "S", "NE", "SE", "none", ""
       usa_market_name: true             # Check if the value is a valid USA market name. Example: "New York, NY"
 
+  - name: "another_column"
+
 ```
 
 
@@ -370,15 +377,16 @@ columns:
 
 ```json
 {
-    "csv"     : {
+    "filename_pattern" : "/demo(-\\d+)?\\.csv$/",
+    "csv"              : {
         "header"     : true,
         "delimiter"  : ",",
         "quote_char" : "\\",
         "enclosure"  : "\"",
         "encoding"   : "utf-8",
         "bom"        : false
     },
-    "columns" : [
+    "columns"          : [
         {
             "name"        : "csv_header_name",
             "description" : "Lorem ipsum",
@@ -412,7 +420,8 @@ columns:
                 "cardinal_direction" : true,
                 "usa_market_name"    : true
             }
-        }
+        },
+        {"name" : "another_column"}
     ]
 }
 
@@ -422,6 +431,7 @@ columns:
 
 
 
+
 <details>
   <summary>Click to see: PHP Format</summary>
 
@@ -430,6 +440,8 @@ columns:
 declare(strict_types=1);
 
 return [
+    'filename_pattern' => '/demo(-\\d+)?\\.csv$/',
+
     'csv' => [
         'header'     => true,
         'delimiter'  => ',',
@@ -438,6 +450,7 @@ return [
         'encoding'   => 'utf-8',
         'bom'        => false,
     ],
+
     'columns' => [
         [
             'name'        => 'csv_header_name',
@@ -473,6 +486,7 @@ return [
                 'usa_market_name'    => true,
             ],
         ],
+        ['name' => 'another_column'],
     ],
 ];
 
@@ -481,6 +495,7 @@ return [
 </details>
 
 
+
 ## Coming soon
 
 It's random ideas and plans. No orderings and deadlines. <u>But batch processing is the priority #1</u>.

schema-examples/full.json

@@ -1,13 +1,14 @@
 {
-    "csv"     : {
+    "filename_pattern" : "/demo(-\\d+)?\\.csv$/",
+    "csv"              : {
         "header"     : true,
         "delimiter"  : ",",
         "quote_char" : "\\",
         "enclosure"  : "\"",
         "encoding"   : "utf-8",
         "bom"        : false
     },
-    "columns" : [
+    "columns"          : [
         {
             "name"        : "csv_header_name",
             "description" : "Lorem ipsum",
@@ -41,6 +42,7 @@
                 "cardinal_direction" : true,
                 "usa_market_name"    : true
             }
-        }
+        },
+        {"name" : "another_column"}
     ]
 }

schema-examples/full.php

@@ -15,6 +15,8 @@
 declare(strict_types=1);
 
 return [
+    'filename_pattern' => '/demo(-\\d+)?\\.csv$/',
+
     'csv' => [
         'header'     => true,
         'delimiter'  => ',',
@@ -23,6 +25,7 @@
         'encoding'   => 'utf-8',
         'bom'        => false,
     ],
+
     'columns' => [
         [
             'name'        => 'csv_header_name',
@@ -58,5 +61,6 @@
                 'usa_market_name'    => true,
             ],
         ],
+        ['name' => 'another_column'],
     ],
 ];

schema-examples/full.yml

@@ -12,6 +12,11 @@
 
 # It's a full example of the CSV schema file in YAML format.
 
+# Regular expression to match the file name. If not set, then no pattern check
+# This way you can validate the file name before the validation process.
+# Feel free to check parent directories as well.
+filename_pattern: /demo(-\d+)?\.csv$/
+
 csv: # Here are default values. You can skip this section if you don't need to override the default values
   header: true                          # If the first row is a header. If true, name of each column is required
   delimiter: ,                          # Delimiter character in CSV file
@@ -66,3 +71,5 @@ columns:
       is_longitude: true                # Can be integer or float. Example: -89.123456
       cardinal_direction: true          # Valid cardinal direction. Examples: "N", "S", "NE", "SE", "none", ""
       usa_market_name: true             # Check if the value is a valid USA market name. Example: "New York, NY"
+
+  - name: "another_column"

src/Csv/CsvFile.php

@@ -82,7 +82,9 @@ public function validate(bool $quickStop = false): ErrorSuite
     {
         $errors = new ErrorSuite($this->getCsvFilename());
 
-        $errors->addErrorSuit($this->validateHeader())
+        $errors
+            ->addErrorSuit($this->validateFile($quickStop))
+            ->addErrorSuit($this->validateHeader($quickStop))
             ->addErrorSuit($this->validateEachCell($quickStop))
             ->addErrorSuit(self::validateAggregateRules($quickStop));
 
@@ -106,7 +108,7 @@ private function prepareReader(): LeagueReader
         return $reader;
     }
 
-    private function validateHeader(): ErrorSuite
+    private function validateHeader(bool $quickStop = false): ErrorSuite
     {
         $errors = new ErrorSuite();
 
@@ -125,6 +127,10 @@ private function validateHeader(): ErrorSuite
 
                 $errors->addError($error);
             }
+
+            if ($quickStop && $errors->count() > 0) {
+                return $errors;
+            }
         }
 
         return $errors;
@@ -152,6 +158,33 @@ private function validateEachCell(bool $quickStop = false): ErrorSuite
         return $errors;
     }
 
+    private function validateFile(bool $quickStop = false): ErrorSuite
+    {
+        $errors = new ErrorSuite();
+
+        $filenamePattern = $this->schema->getFilenamePattern();
+        if (
+            $filenamePattern !== null
+            && $filenamePattern !== ''
+            && \preg_match($filenamePattern, $this->csvFilename) === 0
+        ) {
+            $error = new Error(
+                'filename_pattern',
+                "Filename \"<c>{$this->csvFilename}</c>\" does not match pattern: \"<c>{$filenamePattern}</c>\"",
+                $this->csvFilename,
+                0,
+            );
+
+            $errors->addError($error);
+
+            if ($quickStop && $errors->count() > 0) {
+                return $errors;
+            }
+        }
+
+        return $errors;
+    }
+
     private static function validateAggregateRules(bool $quickStop = false): ErrorSuite
     {
         $errors = new ErrorSuite();

src/Schema.php

@@ -114,9 +114,9 @@ public function getColumn(int|string $columNameOrId): ?Column
         return $column;
     }
 
-    public function getFinenamePattern(): ?string
+    public function getFilenamePattern(): ?string
     {
-        return $this->data->getStringNull('finename_pattern');
+        return Utils::prepareRegex($this->data->getStringNull('filename_pattern'));
     }
 
     public function getIncludes(): array

tests/Blueprint/SchemaTest.php

@@ -44,10 +44,10 @@ public function testFilename(): void
     public function testGetFinenamePattern(): void
     {
         $schemaEmpty = new Schema(self::SCHEMA_EXAMPLE_EMPTY);
-        isSame(null, $schemaEmpty->getFinenamePattern());
+        isSame(null, $schemaEmpty->getFilenamePattern());
 
         $schemaFull = new Schema(self::SCHEMA_EXAMPLE_FULL);
-        isSame('^example\.csv$', $schemaFull->getFinenamePattern());
+        isSame('/^example\.csv$/u', $schemaFull->getFilenamePattern());
     }
 
     public function testScvStruture(): void

tests/Blueprint/ValidatorTest.php

@@ -564,6 +564,25 @@ public function testGetAvaiableRenderFormats(): void
         ], ErrorSuite::getAvaiableRenderFormats());
     }
 
+    public function testFilenamePattern(): void
+    {
+        $csv = new CsvFile(self::CSV_COMPLEX, ['filename_pattern' => '/demo(-\\d+)?\\.csv$/']);
+        isSame(
+            '"filename_pattern" at line 0, column "./tests/fixtures/complex_header.csv". ' .
+            'Filename "./tests/fixtures/complex_header.csv" does not match pattern: "/demo(-\d+)?\.csv$/".',
+            \strip_tags((string)$csv->validate()->get(0)),
+        );
+
+        $csv = new CsvFile(self::CSV_COMPLEX, ['filename_pattern' => '']);
+        isSame('', (string)$csv->validate());
+
+        $csv = new CsvFile(self::CSV_COMPLEX, ['filename_pattern' => null]);
+        isSame('', (string)$csv->validate());
+
+        $csv = new CsvFile(self::CSV_COMPLEX, ['filename_pattern' => '/.*\.csv$/']);
+        isSame('', (string)$csv->validate());
+    }
+
     private function getRule(?string $columnName, ?string $ruleName, array|bool|float|int|string $options): array
     {
         return ['columns' => [['name' => $columnName, 'rules' => [$ruleName => $options]]]];

tests/schemas/demo_valid.yml

@@ -12,6 +12,8 @@
 
 # This schema is valid because match the CSV file (tests/fixtures/demo.csv) perfectly.
 
+filename_pattern: /demo(-\d+)?\.csv$/
+
 columns:
   - name: Name
     rules:

tests/schemas/example_full.yml

@@ -17,7 +17,7 @@
 # Created:      2023-03-09
 
 # File name pattern to match. If not set, then no pattern check
-finename_pattern: ^example\.csv$
+filename_pattern: ^example\.csv$
 
 
 # Include another schemas