Add support for extra field in schema definitions (#197)

Add an `extra` field to various schema definitions and examples. This optional field allows users to include arbitrary data that can be ignored by the validation tool but used for custom purposes. This change increases flexibility and customizability, enhancing the overall utility of the schema configurations.
JBZoo · May 29, 2024 · 7ceffed · 7ceffed
1 parent 3c27b2f
commit 7ceffed
Show file tree

Hide file tree

Showing 16 changed files with 141 additions and 20 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -18,7 +18,7 @@ root = true
 [*]
 end_of_line = lf
 charset = utf-8
-trim_trailing_whitespace = true
+trim_trailing_whitespace = false
 insert_final_newline = true
 indent_style = space
 indent_size = 4

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -183,7 +183,7 @@ jobs:
           mkdir -pv ./build/composer-test
           cd ./build/composer-test
           composer init --no-interaction --name="test/test"
-          composer require jbzoo/csv-blueprint
+          composer require jbzoo/csv-blueprint:dev-${{ github.event.pull_request.head.ref }}
           ./vendor/bin/csv-blueprint
           ./vendor/bin/csv-blueprint validate-csv --ansi -vvv --csv="../../tests/fixtures/batch/*.csv" --schema="../../tests/schemas/demo_valid.yml"
 

diff --git a/README.md b/README.md
@@ -300,7 +300,8 @@ description: |-                         # Any description of the CSV file. Not u
   supporting a wide range of data validation rules from basic type checks to complex regex validations.
   This example serves as a comprehensive guide for creating robust CSV file validations.
 
-presets:                                # Include another schema and define an alias for it.
+# Include another schema and define an alias for it.
+presets:
   my-preset: ./preset_users.yml         # Define preset alias "my-preset". See README.md for details.
 
 # Regular expression to match the file name. If not set, then no pattern check.
@@ -329,6 +330,14 @@ structural_rules: # Here are default values.
   strict_column_order: true             # Ensure columns in CSV follow the same order as defined in this YML schema. It works only if "csv.header" is true.
   allow_extra_columns: false            # Allow CSV files to have more columns than specified in this YML schema.
 
+# Add any extra data you want. It will be ignored by the tool but available for your own code.
+# You can use any format and store anything. Examples:
+# extra: 'some text'
+# extra: [some, options, here]
+# extra: 42
+extra:
+  - key: "value"
+
 # Description of each column in CSV.
 # It is recommended to present each column in the same order as presented in the CSV file.
 # This will not affect the validator, but will make it easier for you to navigate.
@@ -344,6 +353,14 @@ columns:
     # By default, the column is required. It works only if "csv.header" is true and "structural_rules.allow_extra_columns" is false.
     required: true
 
+    # Add any extra data you want. It will be ignored by the tool but available for your own code.
+    # You can use any format and store anything. Examples:
+    # extra: 'some text'
+    # extra: [some, options, here]
+    # extra: 42
+    extra:
+      - key: "value"
+
     ####################################################################################################################
     # Data validation for each(!) value in the column. Please, see notes in README.md
     # Every rule is optional.
@@ -997,6 +1014,8 @@ columns:
   - name: id
     description: Unique identifier, usually used to denote a primary key in databases.
     example: 12345
+    extra: 
+      custom_key: custom value
     rules:
       not_empty: true
       is_trimmed: true
@@ -1192,6 +1211,8 @@ columns:
   - name: id
     description: 'Unique identifier, usually used to denote a primary key in databases.'
     example: 12345
+    extra:
+      custom_key: 'custom value'
     rules:
       not_empty: true
       is_trimmed: true

diff --git a/schema-examples/full.json b/schema-examples/full.json
@@ -24,13 +24,20 @@
         "allow_extra_columns" : false
     },
 
+    "extra"            : [
+        {"key" : "value"}
+    ],
+
     "columns"          : [
         {
             "preset"          : "my-preset/login",
             "name"            : "Column Name (header)",
             "description"     : "Lorem ipsum",
             "example"         : "Some example",
             "required"        : true,
+            "extra"           : [
+                {"key" : "value"}
+            ],
 
             "rules"           : {
                 "preset"                    : "my-preset/login",

diff --git a/schema-examples/full.php b/schema-examples/full.php
@@ -44,13 +44,16 @@
         'allow_extra_columns' => false,
     ],
 
+    'extra' => [['key' => 'value']],
+
     'columns' => [
         [
             'preset'      => 'my-preset/login',
             'name'        => 'Column Name (header)',
             'description' => 'Lorem ipsum',
             'example'     => 'Some example',
             'required'    => true,
+            'extra'       => [['key' => 'value']],
 
             'rules' => [
                 'preset'           => 'my-preset/login',

diff --git a/schema-examples/full.yml b/schema-examples/full.yml
@@ -22,7 +22,8 @@ description: |-                         # Any description of the CSV file. Not u
   supporting a wide range of data validation rules from basic type checks to complex regex validations.
   This example serves as a comprehensive guide for creating robust CSV file validations.
 
-presets:                                # Include another schema and define an alias for it.
+# Include another schema and define an alias for it.
+presets:
   my-preset: ./preset_users.yml         # Define preset alias "my-preset". See README.md for details.
 
 # Regular expression to match the file name. If not set, then no pattern check.
@@ -51,6 +52,14 @@ structural_rules: # Here are default values.
   strict_column_order: true             # Ensure columns in CSV follow the same order as defined in this YML schema. It works only if "csv.header" is true.
   allow_extra_columns: false            # Allow CSV files to have more columns than specified in this YML schema.
 
+# Add any extra data you want. It will be ignored by the tool but available for your own code.
+# You can use any format and store anything. Examples:
+# extra: 'some text'
+# extra: [some, options, here]
+# extra: 42
+extra:
+  - key: "value"
+
 # Description of each column in CSV.
 # It is recommended to present each column in the same order as presented in the CSV file.
 # This will not affect the validator, but will make it easier for you to navigate.
@@ -66,6 +75,14 @@ columns:
     # By default, the column is required. It works only if "csv.header" is true and "structural_rules.allow_extra_columns" is false.
     required: true
 
+    # Add any extra data you want. It will be ignored by the tool but available for your own code.
+    # You can use any format and store anything. Examples:
+    # extra: 'some text'
+    # extra: [some, options, here]
+    # extra: 42
+    extra:
+      - key: "value"
+
     ####################################################################################################################
     # Data validation for each(!) value in the column. Please, see notes in README.md
     # Every rule is optional.

diff --git a/schema-examples/full_clean.yml b/schema-examples/full_clean.yml
@@ -40,12 +40,17 @@ structural_rules:
   strict_column_order: true
   allow_extra_columns: false
 
+extra:
+  - key: value
+
 columns:
   - preset: my-preset/login
     name: 'Column Name (header)'
     description: 'Lorem ipsum'
     example: 'Some example'
     required: true
+    extra:
+      - key: value
 
     rules:
       preset: my-preset/login

diff --git a/schema-examples/preset_database.yml b/schema-examples/preset_database.yml
@@ -17,6 +17,8 @@ columns:
   - name: id
     description: Unique identifier, usually used to denote a primary key in databases.
     example: 12345
+    extra: 
+      custom_key: custom value
     rules:
       not_empty: true
       is_trimmed: true

diff --git a/src/SchemaDataPrep.php b/src/SchemaDataPrep.php
@@ -44,11 +44,14 @@ final class SchemaDataPrep
             'allow_extra_columns' => false,
         ],
 
+        'extra' => null,
+
         'column' => [
             'name'            => '',
             'description'     => '',
             'example'         => null,
             'required'        => true,
+            'extra'           => null,
             'rules'           => [],
             'aggregate_rules' => [],
         ],

diff --git a/src/Utils.php b/src/Utils.php
@@ -293,6 +293,7 @@ public static function compareArray(
             'rules.contains_any',
             'rules.contains_all',
             'rules.ip_v4_range',
+            'extra',
         ];
 
         foreach ($actualSchema as $key => $value) {

diff --git a/src/Validators/ValidatorSchema.php b/src/Validators/ValidatorSchema.php
@@ -21,6 +21,7 @@
 use JBZoo\CsvBlueprint\Utils;
 use JBZoo\Data\AbstractData;
 
+use function JBZoo\Data\data;
 use function JBZoo\Data\phpArray;
 
 final class ValidatorSchema
@@ -51,6 +52,7 @@ public function validate(bool $quickStop = false): ErrorSuite
             }
         }
 
+        unset($expectedColumn['extra']);
         $errors = $this->validateColumns($expectedColumn, $actualColumns, $quickStop);
         if ($errors->count() > 0) {
             $allErrors->addErrorSuit($errors);
@@ -83,10 +85,15 @@ public static function getExpected(): array
 
     private function getActual(): array
     {
-        $actualColumns = $this->data->findSelf('columns');
+        $actualColumns = $this->data->findSelf('columns')->getArrayCopy();
         $actualMeta = $this->data->remove('columns');
+        unset($actualMeta['extra']);
 
-        return [$actualMeta, $actualColumns];
+        foreach (\array_keys($actualColumns) as $index) {
+            unset($actualColumns[$index]['extra']);
+        }
+
+        return [$actualMeta, data($actualColumns)];
     }
 
     private function validateColumns(
@@ -167,7 +174,12 @@ private static function validateMeta(
 
         $actualMetaAsArray = $actualMeta->getArrayCopy();
         $actualPresets = $actualMetaAsArray['presets'] ?? [];
-        unset($expectedMeta['presets'], $actualMetaAsArray['presets']);
+        unset(
+            $expectedMeta['presets'],
+            $expectedMeta['extra'],
+            $actualMetaAsArray['presets'],
+            $actualMetaAsArray['extra'],
+        );
 
         $metaErrors = Utils::compareArray($expectedMeta, $actualMetaAsArray, 'meta', '.');
 

diff --git a/tests/Commands/CreateSchemaTest.php b/tests/Commands/CreateSchemaTest.php
@@ -919,7 +919,7 @@ public function testBigComplex(): void
                   date_min: '2040-05-29'
                   date_max: '2121-03-23'
                   date_format: Ymd
-                  date_age_min: 16
+                  date_age_min: 15
                   date_age_max: 96
                   postal_code: BR
                   hash: crc32

diff --git a/tests/Commands/DebugSchemaTest.php b/tests/Commands/DebugSchemaTest.php
@@ -50,6 +50,7 @@ public function testDumpWithDefaults(): void
                 description: ''
                 example: ~
                 required: true
+                extra: 123
                 rules:
                   not_empty: true
                   length_min: 4
@@ -61,6 +62,7 @@ public function testDumpWithDefaults(): void
                 description: ''
                 example: ~
                 required: true
+                extra: ~
                 rules:
                   not_empty: true
                   is_capitalize: true
@@ -70,6 +72,9 @@ public function testDumpWithDefaults(): void
                 description: ''
                 example: ~
                 required: true
+                extra:
+                  custom_key: custom_value
+                  custom_key_2: custom_value_2
                 rules:
                   not_empty: true
                   is_float: true
@@ -82,6 +87,9 @@ public function testDumpWithDefaults(): void
                 description: ''
                 example: ~
                 required: true
+                extra:
+                  - 123
+                  - 456
                 rules:
                   not_empty: true
                   date_format: Y-m-d
@@ -91,13 +99,15 @@ public function testDumpWithDefaults(): void
                 description: ''
                 example: ~
                 required: true
+                extra: ~
                 rules:
                   not_empty: true
                   allow_values:
                     - red
                     - green
                     - blue
                 aggregate_rules: []
+            extra: 123
 
             YAML;
 
@@ -117,6 +127,7 @@ public function testDumpHideDefaults(): void
             filename_pattern: '/(demo|demo_.*|demo-.*)\.csv$/'
             columns:
               - name: Name
+                extra: 123
                 rules:
                   not_empty: true
                   length_min: 4
@@ -130,6 +141,9 @@ public function testDumpHideDefaults(): void
                   is_capitalize: true
             
               - name: Float
+                extra:
+                  custom_key: custom_value
+                  custom_key_2: custom_value_2
                 rules:
                   not_empty: true
                   is_float: true
@@ -139,6 +153,9 @@ public function testDumpHideDefaults(): void
                   sum_max: 4692
             
               - name: Birthday
+                extra:
+                  - 123
+                  - 456
                 rules:
                   not_empty: true
                   date_format: Y-m-d
@@ -150,6 +167,7 @@ public function testDumpHideDefaults(): void
                     - red
                     - green
                     - blue
+            extra: 123
 
             YAML;
 
@@ -178,6 +196,8 @@ public function testDumpPresetUsage(): void
               - name: id
                 description: 'Unique identifier, usually used to denote a primary key in databases.'
                 example: 12345
+                extra:
+                  custom_key: 'custom value'
                 rules:
                   not_empty: true
                   is_trimmed: true