1. Home
  2. Documentation
  3. Examples
  4. Compare Schemas Using Diff

Compare Schemas Using Diff

Updated: Jan 26, 2024
View on GitHub
Diff Schema DataPipeline Foundations

This example demonstrates how you can use Data Pipeline to modify and compare schemas. This feature lets you know which entities and fields have been added, modified, and removed.  This can be useful in auditing schemas for change management and generating database migration code.

 

Java Code

package com.northconcepts.datapipeline.foundations.examples.schema;

import com.northconcepts.datapipeline.core.FieldType;
import com.northconcepts.datapipeline.foundations.schema.BooleanFieldDef;
import com.northconcepts.datapipeline.foundations.schema.EntityDef;
import com.northconcepts.datapipeline.foundations.schema.NumericFieldDef;
import com.northconcepts.datapipeline.foundations.schema.RecordFieldDef;
import com.northconcepts.datapipeline.foundations.schema.SchemaDef;
import com.northconcepts.datapipeline.foundations.schema.TemporalFieldDef;
import com.northconcepts.datapipeline.foundations.schema.TextFieldDef;
import com.northconcepts.datapipeline.foundations.schema.diff.SchemaDefDiff;

public class CompareSchemas {

    public static void main(String[] args) {
        SchemaDef oldSchema = new SchemaDef("schema-01");

        oldSchema.addEntity(new EntityDef("account")
                .addField(new TextFieldDef("name", FieldType.STRING).setRequired(true).setAllowBlank(false).setMaximumLength(100))
                .addField(new TextFieldDef("number", FieldType.STRING).setRequired(true).setAllowBlank(false).setMaximumLength(20))
                .addField(new NumericFieldDef("balance", FieldType.BIG_DECIMAL).setRequired(true).setMinimum(0).setMaximum(10_000_000)));

        oldSchema.addEntity(new EntityDef("customer")
                .addField(new TextFieldDef("name", FieldType.STRING).setRequired(true).setAllowBlank(false).setMaximumLength(100))
                .addField(new NumericFieldDef("age", FieldType.INT).setRequired(true).setMinimum(25).setMaximum(75))
                .addField(new NumericFieldDef("balance", FieldType.BIG_DECIMAL))
                .addField(new BooleanFieldDef("active", FieldType.BOOLEAN).setAllowedValues(null))
                .addField(new NumericFieldDef("bonuses", FieldType.DOUBLE).setArray(true))
                .addField(new NumericFieldDef("balanceAge", FieldType.DOUBLE).setArray(true))
                .addField(new TemporalFieldDef("lastUpdated", FieldType.DATE).setPattern("yyyy-MM-dd"))
                .addField(new RecordFieldDef("accounts", "account").setRequired(true).setArray(true)));

        SchemaDef newSchema = oldSchema.clone();
        EntityDef customerEntity = newSchema.getEntity("customer");

        // Update property of FieldDef
        customerEntity.getField("age").setRequired(false);

        // Remove field from customer entity
        customerEntity.getFields().remove(customerEntity.indexOfField("balanceAge"));

        // Add a new field to cutomer entity
        customerEntity.addField(new TemporalFieldDef("created_on", FieldType.DATETIME).setPattern("yyyy-MM-dd HH:mm:ss"));

        // Remove account entity
        newSchema.getEntities().remove(newSchema.indexOfEntity("account"));

        // Add new entity to schema
        newSchema.addEntity(new EntityDef("bank")
                .addField(new TextFieldDef("bank_name", FieldType.STRING).setRequired(true))
                );

        SchemaDefDiff diff = SchemaDefDiff.diff(oldSchema, newSchema);
        // All of above updates to newSchema are reported as
        // 1. CHANGED - if property is just updated.
        // 2. ADDED - if a new property is added.
        // 3. REMOVED - if an existing property is removed.
        // 4. NONE - if there is no change for specific property.
        System.out.println("Schema Diff: " + diff);
    }
}

 

Code Walkthrough

  1. An oldSchema is created and initialized with two entities (account and customer) containing a variety of fields along with their properties. 
  2. A newSchema is created by cloning the oldSchema.  The customer entity in the cloned schema is modified to change, remove, and add new fields.  The cloned schema is also modified by removing the account entity and adding a new bank entity.
  3. SchemaDefDiff instance is created by calling diff() method with two arguments of oldSchema and newSchema.
  4. All of the above updates to newSchema are reported as:
    1. CHANGED - if the property is just updated.
    2. ADDED - if a new property is added.
    3. REMOVED - if an existing property is removed.
    4. NONE - if there is no change for the specific property.

 

Console Output

Schema Diff: {
  "children" : [ {
    "name" : "__class__",
    "type" : "NONE"
  }, {
    "name" : "name",
    "type" : "NONE"
  }, {
    "name" : "description",
    "type" : "NONE"
  }, {
    "name" : "account",
    "type" : "REMOVED"
  }, {
    "children" : [ {
      "name" : "name",
      "type" : "NONE"
    }, {
      "name" : "description",
      "type" : "NONE"
    }, {
      "name" : "superEntityName",
      "type" : "NONE"
    }, {
      "name" : "allowExtraFieldsInValidation",
      "type" : "NONE"
    }, {
      "name" : "allowExtraFieldsInMapping",
      "type" : "NONE"
    }, {
      "name" : "addMissingOptionalFields",
      "type" : "NONE"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "NONE"
      }, {
        "name" : "minimumLength",
        "type" : "NONE"
      }, {
        "name" : "maximumLength",
        "type" : "NONE"
      }, {
        "name" : "allowBlank",
        "type" : "NONE"
      }, {
        "name" : "pattern",
        "type" : "NONE"
      } ],
      "name" : "name",
      "type" : "NONE"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "CHANGED"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "NONE"
      }, {
        "name" : "precision",
        "type" : "NONE"
      }, {
        "name" : "scale",
        "type" : "NONE"
      }, {
        "name" : "minimum",
        "type" : "NONE"
      }, {
        "name" : "maximum",
        "type" : "NONE"
      }, {
        "name" : "pattern",
        "type" : "NONE"
      } ],
      "name" : "age",
      "type" : "CHANGED"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "NONE"
      }, {
        "name" : "precision",
        "type" : "NONE"
      }, {
        "name" : "scale",
        "type" : "NONE"
      }, {
        "name" : "minimum",
        "type" : "NONE"
      }, {
        "name" : "maximum",
        "type" : "NONE"
      }, {
        "name" : "pattern",
        "type" : "NONE"
      } ],
      "name" : "balance",
      "type" : "NONE"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "NONE"
      } ],
      "name" : "active",
      "type" : "NONE"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "NONE"
      }, {
        "name" : "precision",
        "type" : "NONE"
      }, {
        "name" : "scale",
        "type" : "NONE"
      }, {
        "name" : "minimum",
        "type" : "NONE"
      }, {
        "name" : "maximum",
        "type" : "NONE"
      }, {
        "name" : "pattern",
        "type" : "NONE"
      } ],
      "name" : "bonuses",
      "type" : "NONE"
    }, {
      "name" : "balanceAge",
      "type" : "REMOVED"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "CHANGED"
      }, {
        "name" : "pattern",
        "type" : "NONE"
      }, {
        "name" : "lenientPattern",
        "type" : "NONE"
      }, {
        "name" : "minimum",
        "type" : "NONE"
      }, {
        "name" : "minimumClass",
        "type" : "NONE"
      }, {
        "name" : "maximum",
        "type" : "NONE"
      }, {
        "name" : "maximumClass",
        "type" : "NONE"
      } ],
      "name" : "lastUpdated",
      "type" : "CHANGED"
    }, {
      "children" : [ {
        "name" : "name",
        "type" : "NONE"
      }, {
        "name" : "description",
        "type" : "NONE"
      }, {
        "name" : "type",
        "type" : "NONE"
      }, {
        "name" : "displayName",
        "type" : "NONE"
      }, {
        "name" : "required",
        "type" : "NONE"
      }, {
        "name" : "array",
        "type" : "NONE"
      }, {
        "name" : "minimumElements",
        "type" : "NONE"
      }, {
        "name" : "maximumElements",
        "type" : "NONE"
      }, {
        "name" : "strictArrays",
        "type" : "NONE"
      }, {
        "name" : "position",
        "type" : "NONE"
      }, {
        "name" : "limitToAllowedValues",
        "type" : "NONE"
      }, {
        "name" : "example",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyPosition",
        "type" : "NONE"
      }, {
        "name" : "primaryKeyType",
        "type" : "NONE"
      }, {
        "name" : "defaultValueExpression",
        "type" : "NONE"
      }, {
        "name" : "indexInEntityDef",
        "type" : "CHANGED"
      }, {
        "name" : "referencedEntityName",
        "type" : "NONE"
      } ],
      "name" : "accounts",
      "type" : "CHANGED"
    }, {
      "name" : "created_on",
      "type" : "ADDED"
    } ],
    "name" : "customer",
    "type" : "CHANGED"
  }, {
    "name" : "bank",
    "type" : "ADDED"
  } ],
  "name" : "schema-01",
  "type" : "CHANGED"
}
Get DataPipeline 9.1.0
All Tags | All Examples
Mobile Analytics