1. Home
  2. Documentation
  3. Examples
  4. Validate Data using Nested Entities

Validate Data using Nested Entities

Updated: Aug 24, 2023
View on GitHub
DataPipeline Foundations Validate Schema

This example validates records that contain nested entities based on predefined rules and definitions. It allows you to define validation rules for both the outer record and its nested records, which can be represented as an array or hierarchical structure. This ensures comprehensive validation of data, considering the integrity and consistency of both the main record and its nested entities.

This can be used during data transformation tasks where the structure of the data is changed, and nested entities are involved. You can define validation rules to check the integrity and validity of both the main record and its nested entities before and after the transformation process. This helps in maintaining data consistency and accuracy throughout the transformation pipeline.

 

Java Code

package com.northconcepts.datapipeline.foundations.examples.schema;

import com.northconcepts.datapipeline.core.FieldType;
import com.northconcepts.datapipeline.core.Record;
import com.northconcepts.datapipeline.foundations.schema.EntityDef;
import com.northconcepts.datapipeline.foundations.schema.NumericFieldDef;
import com.northconcepts.datapipeline.foundations.schema.RecordFieldDef;
import com.northconcepts.datapipeline.foundations.schema.SchemaDef;
import com.northconcepts.datapipeline.foundations.schema.TextFieldDef;
import com.northconcepts.datapipeline.foundations.schema.ValidationMessage;
import com.northconcepts.datapipeline.foundations.schema.ValidationResult;

public class ValidateDataUsingNestedEntities {

    public static void main(String[] args) {
        SchemaDef schema = new SchemaDef();

        schema.addEntity(new EntityDef("products")
                .addField(new NumericFieldDef("product_id", FieldType.LONG))
                .addField(new TextFieldDef("product_name", FieldType.STRING)));

        schema.addEntity(new EntityDef("suppliers")
                .setAllowExtraFieldsInValidation(false)
                .addField(new NumericFieldDef("supplier_id", FieldType.LONG).setPrimaryKeyPosition(0))
                .addField(new TextFieldDef("supplier_name", FieldType.STRING).setPrimaryKeyPosition(1))
                .addField(new TextFieldDef("country", FieldType.STRING))
                .addField(new RecordFieldDef("products_record_field", "products").setArray(true).setMaximumElements(5)));

        Record magazineProduct = new Record()
                .setField("product_id", 1) // This field should be LONG instead of INT
                .setField("product_name", "Magazine");

        Record newsPaperProduct = new Record()
                .setField("product_id", 1) // This field should be LONG instead of INT
                .setField("product_name", "News Paper");

        Record suppierRecord = new Record()
                .setField("supplier_id", 1) // This field should be LONG instead of INT
                .setField("supplier_name", "A1")
                .setField("country", "USA")
                .setField("city", "New York") // Unexpected field
                .setField("products_record_field", new Record[] { magazineProduct, newsPaperProduct });

        System.out.println("==========================Input Record====================================");
        System.out.println(suppierRecord);
        System.out.println("=========================================================================================");

        ValidationResult validationResult = schema.getEntity("suppliers").validateRecord(suppierRecord);
        System.out.println(validationResult);

        System.out.println("=========================================================================================");
        for (int i = 0; i < validationResult.getErrors().size(); i++) {
            ValidationMessage validationMessage = validationResult.getErrors().get(i);
            System.out.println(i + " - " + validationMessage.getMessage());
        }

    }
}

 

Code Walkthrough

  1. SchemaDef instance is created.
  2. A new entity called products is defined and two fields are added to it: product_id (long), product_name (string).
  3. Five fields with corresponding data types are assigned to another new suppliers entity: supplier_id, supplier_name, country, products_record_field.
  4. Two Records with the same fields as the products entity are created: magazineProduct, newsPaperProduct.
  5. Next, another Record instance called supplierRecord is created with five fields and values, and printed in the console. The last field products_record_field of this object contains two nested entity records created in the previous step.
  6. The supplierRecord instance is validated based on the suppliers entity definition and the result is stored in ValidationResult.
  7. Finally, validationResult and each of its validation messages are printed in the console.

 

Console Output

==========================Input Record====================================
Record (MODIFIED) (has child records) {
    0:[supplier_id]:INT=[1]:Integer
    1:[supplier_name]:STRING=[A1]:String
    2:[country]:STRING=[USA]:String
    3:[city]:STRING=[New York]:String
    4:[products_record_field]:ARRAY of RECORD=[[
        Record (MODIFIED) (is child record) {
            0:[product_id]:INT=[1]:Integer
            1:[product_name]:STRING=[Magazine]:String
        }, 
        Record (MODIFIED) (is child record) {
            0:[product_id]:INT=[1]:Integer
            1:[product_name]:STRING=[News Paper]:String
        }]]:ArrayValue
}

=========================================================================================
{
  "errors" : [ {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : false,
      "attributes" : { },
      "fields" : [ {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.supplier_id",
        "limitToAllowedValues" : false,
        "name" : "supplier_id",
        "primaryKey" : true,
        "primaryKeyPosition" : 0,
        "qualifiedName" : "suppliers.supplier_id",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "LONG"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.supplier_name",
        "limitToAllowedValues" : false,
        "name" : "supplier_name",
        "primaryKey" : true,
        "primaryKeyPosition" : 1,
        "qualifiedName" : "suppliers.supplier_name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.country",
        "limitToAllowedValues" : false,
        "name" : "country",
        "primaryKey" : false,
        "qualifiedName" : "suppliers.country",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : true,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.products_record_field",
        "limitToAllowedValues" : false,
        "maximumElements" : 5,
        "minimumElements" : 0,
        "name" : "products_record_field",
        "primaryKey" : false,
        "qualifiedName" : "suppliers.products_record_field",
        "referencedEntityName" : "products",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "RECORD"
      } ],
      "name" : "suppliers",
      "qualifiedName" : "suppliers",
      "tags" : { }
    },
    "field" : {
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "suppliers.supplier_id",
      "limitToAllowedValues" : false,
      "name" : "supplier_id",
      "primaryKey" : true,
      "primaryKeyPosition" : 0,
      "qualifiedName" : "suppliers.supplier_id",
      "required" : false,
      "strictArrays" : true,
      "tags" : { },
      "type" : "LONG"
    },
    "message" : "supplier_id is wrong type; expected LONG, found INT"
  }, {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : true,
      "attributes" : { },
      "fields" : [ {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "products.product_id",
        "limitToAllowedValues" : false,
        "name" : "product_id",
        "primaryKey" : false,
        "qualifiedName" : "products.product_id",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "LONG"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "products.product_name",
        "limitToAllowedValues" : false,
        "name" : "product_name",
        "primaryKey" : false,
        "qualifiedName" : "products.product_name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      } ],
      "name" : "products",
      "qualifiedName" : "products",
      "tags" : { }
    },
    "field" : {
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "products.product_id",
      "limitToAllowedValues" : false,
      "name" : "product_id",
      "primaryKey" : false,
      "qualifiedName" : "products.product_id",
      "required" : false,
      "strictArrays" : true,
      "tags" : { },
      "type" : "LONG"
    },
    "message" : "products_record_field.[0].product_id is wrong type; expected LONG, found INT"
  }, {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : true,
      "attributes" : { },
      "fields" : [ {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "products.product_id",
        "limitToAllowedValues" : false,
        "name" : "product_id",
        "primaryKey" : false,
        "qualifiedName" : "products.product_id",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "LONG"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "products.product_name",
        "limitToAllowedValues" : false,
        "name" : "product_name",
        "primaryKey" : false,
        "qualifiedName" : "products.product_name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      } ],
      "name" : "products",
      "qualifiedName" : "products",
      "tags" : { }
    },
    "field" : {
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "products.product_id",
      "limitToAllowedValues" : false,
      "name" : "product_id",
      "primaryKey" : false,
      "qualifiedName" : "products.product_id",
      "required" : false,
      "strictArrays" : true,
      "tags" : { },
      "type" : "LONG"
    },
    "message" : "products_record_field.[1].product_id is wrong type; expected LONG, found INT"
  }, {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : false,
      "attributes" : { },
      "fields" : [ {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.supplier_id",
        "limitToAllowedValues" : false,
        "name" : "supplier_id",
        "primaryKey" : true,
        "primaryKeyPosition" : 0,
        "qualifiedName" : "suppliers.supplier_id",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "LONG"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.supplier_name",
        "limitToAllowedValues" : false,
        "name" : "supplier_name",
        "primaryKey" : true,
        "primaryKeyPosition" : 1,
        "qualifiedName" : "suppliers.supplier_name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "allowBlank" : true,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.country",
        "limitToAllowedValues" : false,
        "name" : "country",
        "primaryKey" : false,
        "qualifiedName" : "suppliers.country",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : true,
        "attributes" : { },
        "entityQualifiedName" : "suppliers.products_record_field",
        "limitToAllowedValues" : false,
        "maximumElements" : 5,
        "minimumElements" : 0,
        "name" : "products_record_field",
        "primaryKey" : false,
        "qualifiedName" : "suppliers.products_record_field",
        "referencedEntityName" : "products",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "RECORD"
      } ],
      "name" : "suppliers",
      "qualifiedName" : "suppliers",
      "tags" : { }
    },
    "message" : "Record has unexpected field(s) [city]"
  } ],
  "fieldNames" : [ "supplier_id", "product_id", "product_id", null ],
  "fields" : [ {
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "suppliers.supplier_id",
    "limitToAllowedValues" : false,
    "name" : "supplier_id",
    "primaryKey" : true,
    "primaryKeyPosition" : 0,
    "qualifiedName" : "suppliers.supplier_id",
    "required" : false,
    "strictArrays" : true,
    "tags" : { },
    "type" : "LONG"
  }, {
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "products.product_id",
    "limitToAllowedValues" : false,
    "name" : "product_id",
    "primaryKey" : false,
    "qualifiedName" : "products.product_id",
    "required" : false,
    "strictArrays" : true,
    "tags" : { },
    "type" : "LONG"
  }, {
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "products.product_id",
    "limitToAllowedValues" : false,
    "name" : "product_id",
    "primaryKey" : false,
    "qualifiedName" : "products.product_id",
    "required" : false,
    "strictArrays" : true,
    "tags" : { },
    "type" : "LONG"
  }, null ],
  "messages" : [ "supplier_id is wrong type; expected LONG, found INT", "products_record_field.[0].product_id is wrong type; expected LONG, found INT", "products_record_field.[1].product_id is wrong type; expected LONG, found INT", "Record has unexpected field(s) [city]" ],
  "valid" : false
}
=========================================================================================
0 - supplier_id is wrong type; expected LONG, found INT
1 - products_record_field.[0].product_id is wrong type; expected LONG, found INT
2 - products_record_field.[1].product_id is wrong type; expected LONG, found INT
3 - Record has unexpected field(s) [city]
Get DataPipeline 8.3.0
All Tags | All Examples
Mobile Analytics