Map Data from Source to Target in a Pipeline with Validation
Updated: Feb 21, 2022
package com.northconcepts.datapipeline.foundations.examples.datamapping;
import java.io.File;
import java.math.BigDecimal;
import com.northconcepts.datapipeline.core.DataReader;
import com.northconcepts.datapipeline.core.DataWriter;
import com.northconcepts.datapipeline.core.FieldType;
import com.northconcepts.datapipeline.core.StreamWriter;
import com.northconcepts.datapipeline.csv.CSVReader;
import com.northconcepts.datapipeline.filter.FieldCount;
import com.northconcepts.datapipeline.foundations.datamapping.DataMapping;
import com.northconcepts.datapipeline.foundations.datamapping.DataMappingReader;
import com.northconcepts.datapipeline.foundations.datamapping.FieldMapping;
import com.northconcepts.datapipeline.foundations.schema.EntityDef;
import com.northconcepts.datapipeline.foundations.schema.NumericFieldDef;
import com.northconcepts.datapipeline.foundations.schema.SchemaDef;
import com.northconcepts.datapipeline.foundations.schema.TextFieldDef;
import com.northconcepts.datapipeline.job.Job;
public class MapDataFromSourceToTargetInAPipelineWithValidation {
public static void main(String[] args) {
SchemaDef schema = new SchemaDef()
.addEntity(new EntityDef().setName("Product")
.addField(new TextFieldDef("Title", FieldType.STRING).setRequired(true).setMinimumLength(5).setMaximumLength(100))
// .addField(new NumericFieldDef("Price", FieldType.DOUBLE).setRequired(true).setMinimum(10.0))
.addField(new NumericFieldDef("Price", FieldType.BIG_DECIMAL).setRequired(true).setMinimum(10.0))
.addValidation(new FieldCount(3)));
EntityDef productEntity = schema.getEntity("Product");
DataMapping mapping = new DataMapping()
.setTargetValidationEntity(productEntity) // set optional entity definition to validate against
.setValue("Markup", BigDecimal.valueOf(10.00)) // add a constant to be used in the mapping
.addFieldMapping(new FieldMapping("Title", "coalesce(source.Title, source.Handle)"))
.addFieldMapping(new FieldMapping("Cost", "${source.Variant Price}")) // use ${} since field has space in name
.addFieldMapping(new FieldMapping("Price", "toBigDecimal(target.Cost) + Markup"));
DataReader reader = new CSVReader(new File("data/input/jewelry.csv"))
.setAllowMultiLineText(true)
.setFieldNamesInFirstRow(true);
reader = new DataMappingReader(reader, mapping);
DataWriter writer = StreamWriter.newSystemOutWriter();
Job.run(reader, writer);
}
}
