Validate Records using Fields and Rules

In this example, you will learn how you can use DataPipeline to validate data fields and get a response indicating the validation result. It offers a convenient and customizable solution for ensuring data integrity and accuracy.

Users can utilize the example to validate user-submitted data in applications or forms. It can check for data format, completeness, constraints, or any other specific validation rules, providing feedback to users regarding the validity of their input.

Java Code Listing

package com.northconcepts.datapipeline.foundations.examples.schema;

import com.northconcepts.datapipeline.core.FieldType;
import com.northconcepts.datapipeline.core.Record;
import com.northconcepts.datapipeline.filter.FilterExpression;
import com.northconcepts.datapipeline.foundations.schema.EntityDef;
import com.northconcepts.datapipeline.foundations.schema.NumericFieldDef;
import com.northconcepts.datapipeline.foundations.schema.TextFieldDef;

public class ValidateRecordsUsingFieldsAndRules {

    public static void main(String[] args) {
        EntityDef entityDef = new EntityDef()
                .addField(new TextFieldDef("name", FieldType.STRING).setRequired(false).setAllowBlank(false).setMaximumLength(100))
                .addField(new TextFieldDef("email", FieldType.STRING).setRequired(true).setAllowBlank(false).setMaximumLength(255))
                .addField(new NumericFieldDef("age", FieldType.INT).setRequired(true).setMinimum(25).setMaximum(75))
                .addValidation(new FilterExpression("!contains(email, '@example.com') && year(now()) >= 2019"));
        
        
        System.out.println(entityDef.validateRecord(new Record()
                .setField("age", 75)
                .setField("email", "henry@northpole.com")
                .setField("level", "A1")));
        
        //    "valid" : true
        
        System.out.println("------------------------------------");
        
        entityDef.setAllowExtraFieldsInValidation(false);
        
        System.out.println(entityDef.validateRecord(new Record()
                .setField("age", 75)
                .setField("email", "henry@northpole.com")
                .setField("level", "A1")));
        
        //    "valid" : false
        //    "message" : "Record is missing field(s) [name] and has unexpected field(s) [level]"

        System.out.println("------------------------------------");
        
        System.out.println(entityDef.validateRecord(new Record()
                .setField("name", "")
                .setField("age", 76)
                .setField("email", "jsmith@example.com")));

        //    "valid" : false
        //    "message" : "name is blank; expected at least 1 character(s)"
        //     "message" : "age is too large; expected maximum 75, found 76"
        //     "message" : "Record failed validation rule, expected: record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019"
        
    }

}

Code walkthrough

1. A new EntityDef instance is created with three fields and one validation expression. Validation rules such as string length, minimum and maximum values are applied for this field.
2. FilterExpression instance is used to apply custom validation logic for the entity. In the given example, the validation rule checks if the email field contains "@example.com" and the current year is after 2019.
3. Three Record instances are checked for validation rules defined in EntityDef instance. The validateRecord method of EntityDef object is called for that purpose.
4. The first record meets all rules and hence, the result is "valid": "true".
5. setAllowExtraFieldsInValidation(false) method is used to specify that only fields mentioned in the rules are allowed.
6. The second Record instance is validated. As the "name" field is not found and an extra "level" field is added, this record fails the validation, and a corresponding output message is displayed.
7. The validation turn comes to the final Record instance. As the fields in this record do not match the validation rules of the EntityDef, the result along with error messages are printed in the console.

Console Output

{
  "valid" : true
}
------------------------------------
{
  "errors" : [ {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : false,
      "attributes" : { },
      "fields" : [ {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "name",
        "limitToAllowedValues" : false,
        "maximumLength" : 100,
        "name" : "name",
        "primaryKey" : false,
        "qualifiedName" : "name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "email",
        "limitToAllowedValues" : false,
        "maximumLength" : 255,
        "name" : "email",
        "primaryKey" : false,
        "qualifiedName" : "email",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "age",
        "limitToAllowedValues" : false,
        "maximum" : 75,
        "minimum" : 25,
        "name" : "age",
        "primaryKey" : false,
        "qualifiedName" : "age",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "INT"
      } ],
      "tags" : { },
      "validationsSource" : [ "record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019" ]
    },
    "message" : "Record is missing field(s) [name] and has unexpected field(s) [level]"
  } ],
  "fieldNames" : [ null ],
  "fields" : [ null ],
  "messages" : [ "Record is missing field(s) [name] and has unexpected field(s) [level]" ],
  "valid" : false
}
------------------------------------
{
  "errors" : [ {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : false,
      "attributes" : { },
      "fields" : [ {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "name",
        "limitToAllowedValues" : false,
        "maximumLength" : 100,
        "name" : "name",
        "primaryKey" : false,
        "qualifiedName" : "name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "email",
        "limitToAllowedValues" : false,
        "maximumLength" : 255,
        "name" : "email",
        "primaryKey" : false,
        "qualifiedName" : "email",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "age",
        "limitToAllowedValues" : false,
        "maximum" : 75,
        "minimum" : 25,
        "name" : "age",
        "primaryKey" : false,
        "qualifiedName" : "age",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "INT"
      } ],
      "tags" : { },
      "validationsSource" : [ "record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019" ]
    },
    "field" : {
      "allowBlank" : false,
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "name",
      "limitToAllowedValues" : false,
      "maximumLength" : 100,
      "name" : "name",
      "primaryKey" : false,
      "qualifiedName" : "name",
      "required" : false,
      "strictArrays" : true,
      "tags" : { },
      "type" : "STRING"
    },
    "message" : "name is blank; expected at least 1 character(s)"
  }, {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : false,
      "attributes" : { },
      "fields" : [ {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "name",
        "limitToAllowedValues" : false,
        "maximumLength" : 100,
        "name" : "name",
        "primaryKey" : false,
        "qualifiedName" : "name",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "email",
        "limitToAllowedValues" : false,
        "maximumLength" : 255,
        "name" : "email",
        "primaryKey" : false,
        "qualifiedName" : "email",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "age",
        "limitToAllowedValues" : false,
        "maximum" : 75,
        "minimum" : 25,
        "name" : "age",
        "primaryKey" : false,
        "qualifiedName" : "age",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "INT"
      } ],
      "tags" : { },
      "validationsSource" : [ "record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019" ]
    },
    "field" : {
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "age",
      "limitToAllowedValues" : false,
      "maximum" : 75,
      "minimum" : 25,
      "name" : "age",
      "primaryKey" : false,
      "qualifiedName" : "age",
      "required" : true,
      "strictArrays" : true,
      "tags" : { },
      "type" : "INT"
    },
    "message" : "age is too large; expected maximum 75, found 76"
  }, {
    "message" : "Record failed validation rule, expected: record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019"
  } ],
  "fieldNames" : [ "name", "age", null ],
  "fields" : [ {
    "allowBlank" : false,
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "name",
    "limitToAllowedValues" : false,
    "maximumLength" : 100,
    "name" : "name",
    "primaryKey" : false,
    "qualifiedName" : "name",
    "required" : false,
    "strictArrays" : true,
    "tags" : { },
    "type" : "STRING"
  }, {
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "age",
    "limitToAllowedValues" : false,
    "maximum" : 75,
    "minimum" : 25,
    "name" : "age",
    "primaryKey" : false,
    "qualifiedName" : "age",
    "required" : true,
    "strictArrays" : true,
    "tags" : { },
    "type" : "INT"
  }, null ],
  "messages" : [ "name is blank; expected at least 1 character(s)", "age is too large; expected maximum 75, found 76", "Record failed validation rule, expected: record satisfies expression: !contains(email, '@example.com') && year(now()) >= 2019" ],
  "valid" : false
}