🐛 Fix dataset reference in hook for kedro >= 0.19.0 and add unittest

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+-   :bug: Fix dataset reference in hook for kedro >= 0.19.0
+
 ## [0.2.0] - 2024-04-19
 
 ### Changed

kedro_pandera/framework/hooks/pandera_hook.py

@@ -51,13 +51,11 @@ def before_node_run(  # noqa : PLR0913
     ):
         for name, data in inputs.items():
             if (
-                catalog._data_sets[name].metadata is not None
-                and "pandera" in catalog._data_sets[name].metadata
+                catalog._datasets[name].metadata is not None
+                and "pandera" in catalog._datasets[name].metadata
             ):
                 try:
-                    catalog._data_sets[name].metadata["pandera"]["schema"].validate(
-                        data
-                    )
+                    catalog._datasets[name].metadata["pandera"]["schema"].validate(data)
                 except SchemaError as err:
                     self._logger.error(
                         f"Dataset '{name}' pandera validation failed before running '{node.name}', see details in the error message. "

tests/data/iris.csv

@@ -0,0 +1,151 @@
+sepal_length,sepal_width,petal_length,petal_width,species
+5.1,3.5,1.4,0.2,setosa
+4.9,3.0,1.4,0.2,setosa
+4.7,3.2,1.3,0.2,setosa
+4.6,3.1,1.5,0.2,setosa
+5.0,3.6,1.4,0.2,setosa
+5.4,3.9,1.7,0.4,setosa
+4.6,3.4,1.4,0.3,setosa
+5.0,3.4,1.5,0.2,setosa
+4.4,2.9,1.4,0.2,setosa
+4.9,3.1,1.5,0.1,setosa
+5.4,3.7,1.5,0.2,setosa
+4.8,3.4,1.6,0.2,setosa
+4.8,3.0,1.4,0.1,setosa
+4.3,3.0,1.1,0.1,setosa
+5.8,4.0,1.2,0.2,setosa
+5.7,4.4,1.5,0.4,setosa
+5.4,3.9,1.3,0.4,setosa
+5.1,3.5,1.4,0.3,setosa
+5.7,3.8,1.7,0.3,setosa
+5.1,3.8,1.5,0.3,setosa
+5.4,3.4,1.7,0.2,setosa
+5.1,3.7,1.5,0.4,setosa
+4.6,3.6,1.0,0.2,setosa
+5.1,3.3,1.7,0.5,setosa
+4.8,3.4,1.9,0.2,setosa
+5.0,3.0,1.6,0.2,setosa
+5.0,3.4,1.6,0.4,setosa
+5.2,3.5,1.5,0.2,setosa
+5.2,3.4,1.4,0.2,setosa
+4.7,3.2,1.6,0.2,setosa
+4.8,3.1,1.6,0.2,setosa
+5.4,3.4,1.5,0.4,setosa
+5.2,4.1,1.5,0.1,setosa
+5.5,4.2,1.4,0.2,setosa
+4.9,3.1,1.5,0.1,setosa
+5.0,3.2,1.2,0.2,setosa
+5.5,3.5,1.3,0.2,setosa
+4.9,3.1,1.5,0.1,setosa
+4.4,3.0,1.3,0.2,setosa
+5.1,3.4,1.5,0.2,setosa
+5.0,3.5,1.3,0.3,setosa
+4.5,2.3,1.3,0.3,setosa
+4.4,3.2,1.3,0.2,setosa
+5.0,3.5,1.6,0.6,setosa
+5.1,3.8,1.9,0.4,setosa
+4.8,3.0,1.4,0.3,setosa
+5.1,3.8,1.6,0.2,setosa
+4.6,3.2,1.4,0.2,setosa
+5.3,3.7,1.5,0.2,setosa
+5.0,3.3,1.4,0.2,setosa
+7.0,3.2,4.7,1.4,versicolor
+6.4,3.2,4.5,1.5,versicolor
+6.9,3.1,4.9,1.5,versicolor
+5.5,2.3,4.0,1.3,versicolor
+6.5,2.8,4.6,1.5,versicolor
+5.7,2.8,4.5,1.3,versicolor
+6.3,3.3,4.7,1.6,versicolor
+4.9,2.4,3.3,1.0,versicolor
+6.6,2.9,4.6,1.3,versicolor
+5.2,2.7,3.9,1.4,versicolor
+5.0,2.0,3.5,1.0,versicolor
+5.9,3.0,4.2,1.5,versicolor
+6.0,2.2,4.0,1.0,versicolor
+6.1,2.9,4.7,1.4,versicolor
+5.6,2.9,3.6,1.3,versicolor
+6.7,3.1,4.4,1.4,versicolor
+5.6,3.0,4.5,1.5,versicolor
+5.8,2.7,4.1,1.0,versicolor
+6.2,2.2,4.5,1.5,versicolor
+5.6,2.5,3.9,1.1,versicolor
+5.9,3.2,4.8,1.8,versicolor
+6.1,2.8,4.0,1.3,versicolor
+6.3,2.5,4.9,1.5,versicolor
+6.1,2.8,4.7,1.2,versicolor
+6.4,2.9,4.3,1.3,versicolor
+6.6,3.0,4.4,1.4,versicolor
+6.8,2.8,4.8,1.4,versicolor
+6.7,3.0,5.0,1.7,versicolor
+6.0,2.9,4.5,1.5,versicolor
+5.7,2.6,3.5,1.0,versicolor
+5.5,2.4,3.8,1.1,versicolor
+5.5,2.4,3.7,1.0,versicolor
+5.8,2.7,3.9,1.2,versicolor
+6.0,2.7,5.1,1.6,versicolor
+5.4,3.0,4.5,1.5,versicolor
+6.0,3.4,4.5,1.6,versicolor
+6.7,3.1,4.7,1.5,versicolor
+6.3,2.3,4.4,1.3,versicolor
+5.6,3.0,4.1,1.3,versicolor
+5.5,2.5,4.0,1.3,versicolor
+5.5,2.6,4.4,1.2,versicolor
+6.1,3.0,4.6,1.4,versicolor
+5.8,2.6,4.0,1.2,versicolor
+5.0,2.3,3.3,1.0,versicolor
+5.6,2.7,4.2,1.3,versicolor
+5.7,3.0,4.2,1.2,versicolor
+5.7,2.9,4.2,1.3,versicolor
+6.2,2.9,4.3,1.3,versicolor
+5.1,2.5,3.0,1.1,versicolor
+5.7,2.8,4.1,1.3,versicolor
+6.3,3.3,6.0,2.5,virginica
+5.8,2.7,5.1,1.9,virginica
+7.1,3.0,5.9,2.1,virginica
+6.3,2.9,5.6,1.8,virginica
+6.5,3.0,5.8,2.2,virginica
+7.6,3.0,6.6,2.1,virginica
+4.9,2.5,4.5,1.7,virginica
+7.3,2.9,6.3,1.8,virginica
+6.7,2.5,5.8,1.8,virginica
+7.2,3.6,6.1,2.5,virginica
+6.5,3.2,5.1,2.0,virginica
+6.4,2.7,5.3,1.9,virginica
+6.8,3.0,5.5,2.1,virginica
+5.7,2.5,5.0,2.0,virginica
+5.8,2.8,5.1,2.4,virginica
+6.4,3.2,5.3,2.3,virginica
+6.5,3.0,5.5,1.8,virginica
+7.7,3.8,6.7,2.2,virginica
+7.7,2.6,6.9,2.3,virginica
+6.0,2.2,5.0,1.5,virginica
+6.9,3.2,5.7,2.3,virginica
+5.6,2.8,4.9,2.0,virginica
+7.7,2.8,6.7,2.0,virginica
+6.3,2.7,4.9,1.8,virginica
+6.7,3.3,5.7,2.1,virginica
+7.2,3.2,6.0,1.8,virginica
+6.2,2.8,4.8,1.8,virginica
+6.1,3.0,4.9,1.8,virginica
+6.4,2.8,5.6,2.1,virginica
+7.2,3.0,5.8,1.6,virginica
+7.4,2.8,6.1,1.9,virginica
+7.9,3.8,6.4,2.0,virginica
+6.4,2.8,5.6,2.2,virginica
+6.3,2.8,5.1,1.5,virginica
+6.1,2.6,5.6,1.4,virginica
+7.7,3.0,6.1,2.3,virginica
+6.3,3.4,5.6,2.4,virginica
+6.4,3.1,5.5,1.8,virginica
+6.0,3.0,4.8,1.8,virginica
+6.9,3.1,5.4,2.1,virginica
+6.7,3.1,5.6,2.4,virginica
+6.9,3.1,5.1,2.3,virginica
+5.8,2.7,5.1,1.9,virginica
+6.8,3.2,5.9,2.3,virginica
+6.7,3.3,5.7,2.5,virginica
+6.7,3.0,5.2,2.3,virginica
+6.3,2.5,5.0,1.9,virginica
+6.5,3.0,5.2,2.0,virginica
+6.2,3.4,5.4,2.3,virginica
+5.9,3.0,5.1,1.8,virginica

tests/data/iris_schema.yml

@@ -0,0 +1,85 @@
+_example_iris_data_schema:
+  schema_type: dataframe
+  version: 0.18.3
+  columns:
+    sepal_length:
+      title: null
+      description: null
+      dtype: float64
+      nullable: false
+      checks:
+        greater_than_or_equal_to: 4.3
+        less_than_or_equal_to: 7.9
+      unique: false
+      coerce: false
+      required: true
+      regex: false
+    sepal_width:
+      title: null
+      description: null
+      dtype: float64
+      nullable: false
+      checks:
+        greater_than_or_equal_to: 2.0
+        less_than_or_equal_to: 4.4
+      unique: false
+      coerce: false
+      required: true
+      regex: false
+    petal_length:
+      title: null
+      description: null
+      dtype: float64
+      nullable: false
+      checks:
+        greater_than_or_equal_to: 1.0
+        less_than_or_equal_to: 6.9
+      unique: false
+      coerce: false
+      required: true
+      regex: false
+    petal_width:
+      title: null
+      description: null
+      dtype: float64
+      nullable: false
+      checks:
+        greater_than_or_equal_to: 0.1
+        less_than_or_equal_to: 2.5
+      unique: false
+      coerce: false
+      required: true
+      regex: false
+    species:
+      title: null
+      description: null
+      dtype: object
+      nullable: false
+      checks: null
+      unique: false
+      coerce: false
+      required: true
+      regex: false
+  checks: null
+  index:
+  - title: null
+    description: null
+    dtype: int64
+    nullable: false
+    checks:
+      greater_than_or_equal_to: 0.0
+      less_than_or_equal_to: 149.0
+    name: null
+    unique: false
+    coerce: false
+  dtype: null
+  coerce: true
+  strict: false
+  name: null
+  ordered: false
+  unique: null
+  report_duplicates: all
+  unique_column_names: false
+  add_missing_columns: false
+  title: null
+  description: null

tests/framework/hooks/test_hook.py

@@ -0,0 +1,30 @@
+from kedro.io import DataCatalog
+from kedro.pipeline import node
+from kedro_datasets.pandas import CSVDataset
+from pandera.io import from_yaml
+
+from kedro_pandera.framework.hooks.pandera_hook import PanderaHook
+
+
+def test_hook():
+    test_schema = from_yaml("tests/data/iris_schema.yml")
+    test_catalog = DataCatalog(
+        {
+            "iris": CSVDataset(
+                filepath="tests/data/iris.csv",
+                metadata={"pandera": {"schema": test_schema}},
+            )
+        }
+    )
+    test_hook = PanderaHook()
+    test_inputs = {"iris": test_catalog.load("iris")}
+    test_node = node(
+        name="test_node", func=lambda iris: True, inputs=["iris"], outputs=None
+    )
+    test_hook.before_node_run(
+        node=test_node,
+        catalog=test_catalog,
+        inputs=test_inputs,
+        is_async=False,
+        session_id=0,
+    )