Map Records using Schema

This example shows how you can use Data Pipeline to map records based on the entity specified in a schema. It provides a flexible and customizable approach to map and transform data records according to a defined entity structure.  This allows users to achieve data consistency and compatibility across different systems and formats.

Java Code Listing

package com.northconcepts.datapipeline.foundations.examples.schema;

import com.northconcepts.datapipeline.core.FieldType;
import com.northconcepts.datapipeline.core.Record;
import com.northconcepts.datapipeline.foundations.schema.BooleanFieldDef;
import com.northconcepts.datapipeline.foundations.schema.EntityDef;
import com.northconcepts.datapipeline.foundations.schema.NumericFieldDef;
import com.northconcepts.datapipeline.foundations.schema.TemporalFieldDef;
import com.northconcepts.datapipeline.foundations.schema.TextFieldDef;

public class MapRecordUsingSchema {

    public static void main(String[] args) {
        Record record = new Record();
        record.addField("name", "John Smith");
        record.addField("age", "76");
        record.addField("balance", "31.05");
        record.addField("active", "true"); // "yes" and non-zero numbers also map to true
        record.addField("bonuses", new String[]{"100", "500", "2000"});
        record.addField("balanceAge", new double[]{0, 30, 1.05});
        record.addField("lastUpdated", "2019-12-19");

        EntityDef entityDef = new EntityDef();
        entityDef.addField(new TextFieldDef("name", FieldType.STRING).setRequired(true).setAllowBlank(false).setMaximumLength(100));
        entityDef.addField(new NumericFieldDef("age", FieldType.INT).setRequired(true).setMinimum(25).setMaximum(75));
        entityDef.addField(new NumericFieldDef("balance", FieldType.BIG_DECIMAL));
        entityDef.addField(new BooleanFieldDef("active", FieldType.BOOLEAN).setAllowedValues(null));
        entityDef.addField(new NumericFieldDef("bonuses", FieldType.DOUBLE).setArray(true));
        entityDef.addField(new NumericFieldDef("balanceAge", FieldType.DOUBLE).setArray(true));
        entityDef.addField(new TemporalFieldDef("lastUpdated", FieldType.DATE).setPattern("yyyy-MM-dd"));


        System.out.println("Original Record-----------------------------------");
        System.out.println(record);
        System.out.println("Mapping Result-----------------------------------");
        System.out.println(entityDef.mapRecord(record));  // map record in place
        System.out.println("Mapped Record-----------------------------------");
        System.out.println(record);
        System.out.println("Validation Result-----------------------------------");
        System.out.println(entityDef.validateRecord(record));
    }

}

Code Walkthrough

1. A new Record instance is created with 7 fields.
2. A new entity is created with the same field names as the Record. Additional data types and validation rules are specified in the entity definition.
3. The record object is printed in the console with System.out.println().
4. Next, the record is mapped based on the entity and the result is printed in the console.
5. The validation report/message of the record is also shown in the console.

Console output

Original Record-----------------------------------
Record (MODIFIED) {
    0:[name]:STRING=[John Smith]:String
    1:[age]:STRING=[76]:String
    2:[balance]:STRING=[31.05]:String
    3:[active]:STRING=[true]:String
    4:[bonuses]:ARRAY of STRING=[[100, 500, 2000]]:ArrayValue
    5:[balanceAge]:ARRAY of DOUBLE=[[0.0, 30.0, 1.05]]:ArrayValue
    6:[lastUpdated]:STRING=[2019-12-19]:String
}

Mapping Result-----------------------------------
{
  "valid" : true
}
Mapped Record-----------------------------------
Record (MODIFIED) {
    0:[name]:STRING=[John Smith]:String
    1:[age]:INT=[76]:Integer
    2:[balance]:BIG_DECIMAL=[31.05]:BigDecimal
    3:[active]:BOOLEAN=[true]:Boolean
    4:[bonuses]:ARRAY of DOUBLE=[[100.0, 500.0, 2000.0]]:ArrayValue
    5:[balanceAge]:ARRAY of DOUBLE=[[0.0, 30.0, 1.05]]:ArrayValue
    6:[lastUpdated]:DATE=[2019-12-19]:Date
}

Validation Result-----------------------------------
{
  "errors" : [ {
    "entity" : {
      "addMissingOptionalFields" : false,
      "allowExtraFieldsInMapping" : true,
      "allowExtraFieldsInValidation" : true,
      "attributes" : { },
      "fields" : [ {
        "allowBlank" : false,
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "name",
        "limitToAllowedValues" : false,
        "maximumLength" : 100,
        "name" : "name",
        "primaryKey" : false,
        "qualifiedName" : "name",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "STRING"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "age",
        "limitToAllowedValues" : false,
        "maximum" : 75,
        "minimum" : 25,
        "name" : "age",
        "primaryKey" : false,
        "qualifiedName" : "age",
        "required" : true,
        "strictArrays" : true,
        "tags" : { },
        "type" : "INT"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "balance",
        "limitToAllowedValues" : false,
        "name" : "balance",
        "primaryKey" : false,
        "qualifiedName" : "balance",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "BIG_DECIMAL"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "active",
        "limitToAllowedValues" : false,
        "name" : "active",
        "primaryKey" : false,
        "qualifiedName" : "active",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "BOOLEAN"
      }, {
        "array" : true,
        "attributes" : { },
        "entityQualifiedName" : "bonuses",
        "limitToAllowedValues" : false,
        "minimumElements" : 0,
        "name" : "bonuses",
        "primaryKey" : false,
        "qualifiedName" : "bonuses",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "DOUBLE"
      }, {
        "array" : true,
        "attributes" : { },
        "entityQualifiedName" : "balanceAge",
        "limitToAllowedValues" : false,
        "minimumElements" : 0,
        "name" : "balanceAge",
        "primaryKey" : false,
        "qualifiedName" : "balanceAge",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "DOUBLE"
      }, {
        "array" : false,
        "attributes" : { },
        "entityQualifiedName" : "lastUpdated",
        "lenientPattern" : true,
        "limitToAllowedValues" : false,
        "name" : "lastUpdated",
        "pattern" : "yyyy-MM-dd",
        "primaryKey" : false,
        "qualifiedName" : "lastUpdated",
        "required" : false,
        "strictArrays" : true,
        "tags" : { },
        "type" : "DATE"
      } ],
      "tags" : { }
    },
    "field" : {
      "array" : false,
      "attributes" : { },
      "entityQualifiedName" : "age",
      "limitToAllowedValues" : false,
      "maximum" : 75,
      "minimum" : 25,
      "name" : "age",
      "primaryKey" : false,
      "qualifiedName" : "age",
      "required" : true,
      "strictArrays" : true,
      "tags" : { },
      "type" : "INT"
    },
    "message" : "age is too large; expected maximum 75, found 76"
  } ],
  "fieldNames" : [ "age" ],
  "fields" : [ {
    "array" : false,
    "attributes" : { },
    "entityQualifiedName" : "age",
    "limitToAllowedValues" : false,
    "maximum" : 75,
    "minimum" : 25,
    "name" : "age",
    "primaryKey" : false,
    "qualifiedName" : "age",
    "required" : true,
    "strictArrays" : true,
    "tags" : { },
    "type" : "INT"
  } ],
  "messages" : [ "age is too large; expected maximum 75, found 76" ],
  "valid" : false
}