D-Net

package eu.dnetlib.data.transform;

import java.io.*;

import java.util.*;

import java.util.Map.Entry;

import java.util.stream.Collectors;

import java.util.zip.GZIPInputStream;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactoryConfigurationError;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.google.gson.JsonElement;

import com.google.gson.JsonParser;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.graph.model.DNGFDecoder;

import eu.dnetlib.data.graph.model.DNGFRowKeyDecoder;

import eu.dnetlib.data.graph.utils.RelDescriptor;

import eu.dnetlib.data.mapreduce.hbase.index.config.EntityConfigTable;

import eu.dnetlib.data.mapreduce.hbase.index.config.IndexConfig;

import eu.dnetlib.data.mapreduce.hbase.index.config.IndexConfigTest;

import eu.dnetlib.data.mapreduce.hbase.index.config.LinkDescriptor;

import eu.dnetlib.data.mapreduce.util.UpdateMerger;

import eu.dnetlib.data.mapreduce.util.XmlRecordFactory;

import eu.dnetlib.data.mapreduce.util.XmlRecordFactoryTest;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.DliFieldTypeProtos;

import eu.dnetlib.data.proto.DliProtos;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.functional.xml.XMLIndenter;

import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner;

import eu.dnetlib.pace.config.DedupConfig;

import eu.dnetlib.pace.distance.PaceDocumentDistance;

import eu.dnetlib.pace.distance.eval.ScoreResult;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.ProtoDocumentBuilder;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.*;

public class XsltRowTransformerFactoryTest extends AbstractTransformerTest {

	private static final Log log = LogFactory.getLog(XsltRowTransformerFactoryTest.class);

	private EntityConfigTable entityConfigTable;

	@Before

	public void setUp() throws Exception {

		factory = new XsltRowTransformerFactory();

		entityConfigTable = IndexConfig.load(IndexConfigTest.loadConfiguration()).getConfigMap();

	}

	@Test

	public void testParseManyAuthors() throws Exception {

		final Map<String, Object> xslParams = Maps.newHashMap();

		final Map<String, String> m = Maps.newHashMap();

		m.put("od______2367", "true"); // Puma

		m.put("od______2294", "true"); // UNIBI

		m.put("od________18", "false"); // Arxiv

		xslParams.put("mergeIdForHomonymsMap", m);

		final List<Row> rows = asRows(loadFromTransformationProfile("oaf_person2hbase.xml"), xslParams, load("recordManyAuthors.xml"));

		int authorCount = 0;

		for (final Row row : rows) {

			for (final Column<String, byte[]> col : row.getColumns()) {

				authorCount++;

				final DNGFDecoder d = DNGFDecoder.decode(col.getValue());

				assertNotNull(d);

				log.debug(d.getEntity().getPerson().getCoauthorList().size());

			}

		}

		log.info("authors' count: ---> " + authorCount);

	}

	@Test

	public void testParseAuthors() throws Exception {

		final Map<String, Object> xslParams = Maps.newHashMap();

		final Map<String, String> m = Maps.newHashMap();

		m.put("od______2367", "true"); // Puma

		m.put("od______2294", "true"); // UNIBI

		m.put("od________18", "false"); // Arxiv

		xslParams.put("mergeIdForHomonymsMap", m);

		final List<Row> rows = asRows(loadFromTransformationProfile("oaf_person2hbase.xml"), xslParams, load("recordArxiv.xml"));

		for (final Row row : rows) {

			for (final Column<String, byte[]> col : row.getColumns()) {

				final DNGFDecoder d = DNGFDecoder.decode(col.getValue());

				log.debug(d.getDNGF());

			}

		}

	}

	@Test

	public void testParseDNGFClaimUpdate() throws Exception {

		doTest(loadFromTransformationProfile("claimUpdates_2_hbase.xml"), load("recordClaimUpdate.xml"));

	}

	@Test

	public void testParseClaimUpdate() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("recordClaim.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("claimUpdates_2_hbase.xml"), load("recordClaimUpdate.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testParseDatasetPUB() throws Exception {

		doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("recordDatasetPUB.xml"));

	}

	@Test

	public void testParseClaim() throws Exception {

		doTest(loadFromTransformationProfile("pmf2hbase.xml"), load("recordClaim.xml"));

	}

	@Test

	public void testParseClaimDataset() throws Exception {

		doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("recordClaimDataset.xml"));

	}

	@Test

	public void testParseACM() throws Exception {

		doTest(loadFromTransformationProfile("pmf2hbase.xml"), load("recordACM.xml"));

	}

	@Test

	public void testParseASB() throws Exception {

		doTest(loadFromTransformationProfile("pmf2hbase.xml"), load("recordASB.xml"));

	}

	@Test

	public void testParseProjectCorda() throws Exception {

		doTest(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordCorda.xml"));

	}

	@Test

	public void testParseProjectFCT() throws Exception {

		doTest(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordFCT.xml"));

	}

	@Test

	public void testParseDNGF() throws Exception {

		doTest(loadFromTransformationProfile("pmf2hbase.xml"), load("record.xml"));

	}

	@Test

	public void testParseDNGFPublication() throws Exception {

		doTest(loadFromTransformationProfile("oaf_publication2hbase.xml"), load("record.xml"));

	}

	@Test

	public void testParseWDSDaaaatacite() throws Exception {

        //doTest(loadFromTransformationProfile("dmfwds2hbase.xml"), load("recordWds.xml"));

    }

	@Test

	public void testParseDLIDatacite() throws Exception {

        final List<Row> rows = Lists.newArrayList();

        rows.addAll(asRows(loadFromTransformationProfile("dmfdli2hbase.xml"), load("record_dli_dmf2.xml")));

        rows.forEach(row ->

                row.getColumns().forEach(

						result -> {

							if (result != null) {

								final DNGFDecoder decoder =

										DNGFDecoder.decode(result.getValue(), DliFieldTypeProtos.completionStatus, DliProtos.completionStatus, DliProtos.resolvedfrom, DliProtos.typedIdentifier);

							System.out.println(result.getName());

							System.out.println("decoder.getDNGF().toString() = " + decoder.getDNGF().toString());

							}

						}

				));

	}

	@Test

	public void testParseDLIPMF() throws Exception {

		doTest(loadFromTransformationProfile("pmfdli2hbase.xml"), load("record_dli_pmf.xml"));

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("pmfdli2hbase.xml"), load("record_dli_pmf.xml")));

		rows.forEach(row -> {

			System.out.println(row.getKey().toString());

			row.getColumns().forEach(result -> {

				System.out.println(result.getName());

						if (result != null) {

							final DNGFDecoder decoder =

									DNGFDecoder.decode(result.getValue(), DliFieldTypeProtos.completionStatus, DliProtos.completionStatus, DliProtos.resolvedfrom, DliProtos.typedIdentifier);

							System.out.println("decoder.getDNGF().toString() = " + decoder.getDNGF().toString());

						}

					}

			);

		});

	}

	@Test

	public void testParseDatacite() throws Exception {

		doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("recordDatacite.xml"));

	}

	@Test

	public void testParseDatacite2() throws Exception {

		doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("recordDatacite2.xml"));

	}

	@Test

	public void testParseOpenTrials() throws Exception {

		doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("opentrials_datacite1.xml"));

	}

	@Test

	public void testLinkPangaea() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("dmf2hbase.xml"), load("pangaeODF1.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("dmf2hbase.xml"), load("pangaeODF2.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("pangaeOAF.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordCordaPangaea.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testLinkCorda() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordCorda.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("recordCorda.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testLinkFCT() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordFCT.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("recordFCT.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testLinkARC() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordARC.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("recordARC.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testLinkWT() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordWT.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), load("recordWT.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testLinkOrganization() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("organizations_2_hbase.xml"), load("organization.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("projectorganization_2_hbase.xml"), load("project_organization.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("project_2_hbase.xml"), load("projectRecordCorda.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testUnpackAuthors() throws Exception {

		final Map<String, Object> xslParams = Maps.newHashMap();

		final Map<String, String> m = Maps.newHashMap();

		m.put("od______2367", "true"); // Puma

		m.put("od______2294", "true"); // UNIBI

		m.put("od________18", "false"); // Arxiv

		xslParams.put("mergeIdForHomonymsMap", m);

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), xslParams, load("recordArxiv.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), xslParams, load("recordArxiv2.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), xslParams, load("recordPuma1.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), xslParams, load("recordPuma2.xml")));

		rows.addAll(asRows(loadFromTransformationProfile("pmf2hbase.xml"), xslParams, load("recordUNIBI.xml")));

		printPersonFullnames(mapAll(buildTable(rows)));

	}

	@Test

	public void testParseDoajOAF() throws Exception {

		doTest(loadFromTransformationProfile("pmf2hbase.xml"), load("doajUniversityRecord.xml"));

	}

	@Test

	public void testParseDatasource() throws Exception {

		doTest(loadFromTransformationProfile("datasources_2_hbase.xml"), load("datasourceNative.xml"));

	}

	@Test

	public void testParseDataDatasource() throws Exception {

		doTest(loadFromTransformationProfile("datasources_2_hbase.xml"), load("dataDatasource.xml"));

	}

	@Test

	public void testFromMongodbCompressedDump() throws Exception {

		doTestJsonGz(loadFromTransformationProfile("pmf2hbase.xml"), load("mdstore_cleaned.json.gz"));

	}

	@Test

	public void testLoadFromTransformationProfile() throws IOException {

		InputStream in = loadFromTransformationProfile("pmf2hbase.xml");

		log.info(IOUtils.toString(in));

	}

	@Test

	public void test_template() throws Exception {

		final String xslt = IOUtils.toString(loadFromTransformationProfile("pmf2hbase.xml"));

		final XsltRowTransformer transformer = factory.getTransformer(xslt);

		assertNotNull(transformer);

		final String record = IOUtils.toString(load("record.xml"));

		final List<Row> rows = transformer.apply(record);

		System.out.println(rows);

	}

	@Test

	public void testWrongCharsOrganization() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("organizations_2_hbase.xml"), load("organizationWrongChars.xml")));

		printAll(mapAll(buildTable(rows)));

	}

	@Test

	public void testMapDLIDataciteAsMapDocument() throws Exception {

		final List<Row> rows = Lists.newArrayList();

		rows.addAll(asRows(loadFromTransformationProfile("dmfdli2hbase.xml"), load("record_dli_dmf.xml")));

		final List<Optional<DNGF>> collect = rows.stream().map(row -> row.getColumns().stream().map(

				result -> {

					final DNGFDecoder decoder =

							DNGFDecoder.decode(result.getValue(), DliFieldTypeProtos.completionStatus, DliProtos.completionStatus, DliProtos.resolvedfrom,

									DliProtos.typedIdentifier);

					return decoder.getDNGF();

				}

		).filter(dngf -> dngf.getKind().equals(Kind.entity) && dngf.getEntity().getType().equals(Type.dataset))

				.findFirst())

				.filter(Optional::isPresent)

				.collect(Collectors.toList());

		collect.forEach(c -> {

			System.out.println(c.get().toString());

		});

		final DNGFEntity entity = collect.get(0).get().getEntity();

		final DedupConfig conf = DedupConfig.load(IOUtils.toString(load("/eu/dnetlib/pace/dli.pace.dataset.json")));

		final MapDocument d1 = ProtoDocumentBuilder.newInstance(entity.getId(), entity, conf.getPace().getModel());

		final ScoreResult sr = new PaceDocumentDistance().between(d1, d1, conf);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		BlacklistAwareClusteringCombiner.filterAndCombine(d1, conf, conf.getPace().getBlacklists());

		System.out.println(d1.toString());

	}

	private void doTest(final InputStream xsltStream, final InputStream recordStream) throws Exception {

		try {

			final List<Row> rows = asRows(xsltStream, recordStream);

			log.info(rows);

			final Map<String, Map<String, Map<String, byte[]>>> table = buildTable(rows);

			//System.out.println("\n" + table.toString());

			final Map<String, XmlRecordFactory> builders = mapAll(table);

			printAll(builders);

		} catch (final InvalidProtocolBufferException e) {

			throw new Exception(e);

		} catch (final TransformerConfigurationException e) {

			throw new Exception(e);

		} catch (final TransformerFactoryConfigurationError e) {

			throw new Exception(e);

		} catch (final DocumentException e) {

			throw new Exception(e);

		}

	}

	private void doTestJsonGz(final InputStream xsltStream, final InputStream recordStream) throws Exception {

		final Iterator<List<Row>> rowsIterator = asRowsJsonGzip(xsltStream, recordStream);

		int i = 0;

		while (rowsIterator.hasNext()) {

			final List<Row> rows = rowsIterator.next();

			i++;

			if ((i % 10000) == 0) {

				System.out.println(i);

			}

			final Map<String, Map<String, Map<String, byte[]>>> table = buildTableDoaj(rows);

			for (final Map<String, Map<String, byte[]>> m : table.values()) {

				for (final Map<String, byte[]> mv : m.values()) {

					for (final byte[] v : mv.values()) {

						final DNGFDecoder d = DNGFDecoder.decode(v);

						assertNotNull(d);

						assertNotNull(d.getDNGF());

						switch (d.getKind()) {

						case entity:

							assertNotNull(d.getMetadata());

							final Type type = d.getDNGF().getEntity().getType();

							if (type.equals(Type.publication) || type.equals(Type.dataset)) {

								System.out.println(d.getDNGF());

							}

							break;

						case relation:

							assertNotNull(d.getDNGFRel());

							break;

						default:

							break;

						}

					}

				}

			}

		}

	}

	private Iterator<List<Row>> asRowsJsonGzip(final InputStream xsltStream, final InputStream recordStreamJsonGzip) throws Exception {

		final String xslt = IOUtils.toString(xsltStream);

		final XsltRowTransformer transformer = factory.getTransformer(xslt);

		assertNotNull(transformer);

		assertNotNull(recordStreamJsonGzip);

		final GZIPInputStream stream = new GZIPInputStream(recordStreamJsonGzip);

		assertNotNull(stream);

		final BufferedReader inStream = new BufferedReader(new InputStreamReader(stream));

		assertNotNull(inStream);

		return new Iterator<List<Row>>() {

			String jsonRecord = null;

			@Override

			public boolean hasNext() {

				try {

					return (jsonRecord = inStream.readLine()) != null;

				} catch (final IOException e) {

					throw new RuntimeException(e);

				}

			}

			@Override

			public List<Row> next() {

				final JsonElement json = new JsonParser().parse(jsonRecord);

				final String body = json.getAsJsonObject().get("body").getAsString();

				try {

					assertTrue(StringUtils.isNotBlank(body));

					// System.out.println(body);

					final List<Row> rows = transformer.apply(body);

					assertNotNull(rows);

					assertFalse(rows.isEmpty());

					return rows;

				} catch (final Throwable e) {

					System.err.println("error transforming document: " + body);

					throw new RuntimeException(e);

				}

			}

			@Override

			public void remove() {

				throw new UnsupportedOperationException();

			}

		};

	}

	private Map<String, Map<String, Map<String, byte[]>>> buildTableDoaj(final List<Row> rows) throws UnsupportedEncodingException {

		final Map<String, Map<String, Map<String, byte[]>>> table = Maps.newHashMap();

		for (final Row row : rows) {

			final String rowKey = row.getKey();

			final String cf = row.getColumnFamily();

			if (!table.containsKey(rowKey)) {

				table.put(rowKey, new HashMap<>());

			}

			if (!table.get(rowKey).containsKey(cf)) {

				table.get(rowKey).put(row.getColumnFamily(), new HashMap<>());

			}

			for (final Column<String, byte[]> c : row.getColumns()) {

				// System.out.println(String.format("ADDING K:%s CF:%s Q:%s", rowKey, cf, c.getName()));

				table.get(rowKey).get(cf).put(c.getName(), c.getValue());

				if (cf.equals("result") && c.getName().equals("body")) {

					// System.out.println(String.format("ADDING K:%s CF:%s Q:%s", rowKey, cf, c.getName()));

					assertTrue(StringUtils.isNotBlank(new String(c.getValue(), "UTF-8")));

				}

			}

		}

		return table;

	}

	private Map<String, XmlRecordFactory> mapAll(final Map<String, Map<String, Map<String, byte[]>>> table) throws Exception {

		final Map<String, XmlRecordFactory> builders = Maps.newHashMap();

		for (final Entry<String, Map<String, Map<String, byte[]>>> e : table.entrySet()) {

			map(builders, e.getKey(), e.getValue());

		}

		return builders;

	}

	// private Map<String, XmlRecordFactory> mapResultsOnly(final Map<String, Map<String, Map<String, byte[]>>> table) throws Exception {

	//

	// final Map<String, XmlRecordFactory> builders = Maps.newHashMap();

	// for (final Entry<String, Map<String, Map<String, byte[]>>> e : table.entrySet()) {

	// final Type type = DNGFRowKeyDecoder.decode(e.getKey()).getType();

	// if (type == Type.result) {

	// map(builders, e.getKey(), e.getValue());

	// }

	// }

	// return builders;

	// }

	private void map(final Map<String, XmlRecordFactory> builders, final String rowKey, final Map<String, Map<String, byte[]>> row) throws Exception {

		final Type type = DNGFRowKeyDecoder.decode(rowKey).getType();

		final Map<String, byte[]> familyMap = row.get(type.toString());

		if (familyMap == null) return;

		final byte[] bodyB = familyMap.get("body");

		if (bodyB != null) {

			ensureBuilder(builders, rowKey);

			final DNGF oaf = UpdateMerger.mergeBodyUpdates(familyMap, type);

			final DNGFDecoder mainEntity = DNGFDecoder.decode(oaf);

			builders.get(rowKey).setMainEntity(mainEntity);

			for (final LinkDescriptor ld : entityConfigTable.getDescriptors(type)) {

				final String it = ld.getRelDescriptor().getCode();

				final Map<String, byte[]> cols = row.get(it);

				if ((cols != null) && !cols.isEmpty()) {

					for (final byte[] oafB : cols.values()) {

						final DNGF.Builder relBuilder = DNGF.newBuilder(DNGF.parseFrom(oafB));

						if (ld.isSymmetric()) {

							final RelDescriptor rd = ld.getRelDescriptor();

							relBuilder.getRelBuilder().setCachedTarget(mainEntity.getEntity()).setRelType(rd.asQualifier());

						}

						relBuilder.getRelBuilder().setChild(ld.isChild());

						final DNGF.Builder oafBuilder = DNGF.newBuilder().setKind(Kind.relation).setLastupdatetimestamp(System.currentTimeMillis());

						oafBuilder.mergeFrom(relBuilder.build());

						final String targetId = ld.isSymmetric() ? oafBuilder.getRel().getTarget() : oafBuilder.getRel().getSource();

						ensureBuilder(builders, targetId);

						final DNGFDecoder decoder = DNGFDecoder.decode(oafBuilder.build());

						if (ld.isChild()) {

							builders.get(targetId).addChild(type, decoder);

						} else {

							builders.get(targetId).addRelation(type, decoder);

						}

					}

				}

			}

		}

	}

	private void printAll(final Map<String, XmlRecordFactory> builders) throws DocumentException {

		print(Sets.newHashSet(Type.values()), builders, null);

	}

	private void printPerson(final Map<String, XmlRecordFactory> builders) throws DocumentException {

		print(Sets.newHashSet(Type.person), builders, null);

	}

	private void printPersonFullnames(final Map<String, XmlRecordFactory> builders) throws DocumentException {

		final Map<Type, Set<String>> xpaths = Maps.newHashMap();

		final Set<String> personPaths = Sets.newHashSet();

		personPaths.add("//fullname");

		xpaths.put(Type.person, personPaths);

		print(Sets.newHashSet(Type.person), builders, xpaths);

	}

	private void print(final Set<Type> types, final Map<String, XmlRecordFactory> builders, final Map<Type, Set<String>> xpaths) throws DocumentException {

		final SAXReader r = new SAXReader();

		for (final Entry<String, XmlRecordFactory> e : builders.entrySet()) {

			final DNGFRowKeyDecoder kd = DNGFRowKeyDecoder.decode(e.getKey());

			if (!e.getValue().isValid()) throw new IllegalArgumentException("invalid builder: " + e.getKey());

			if (types.contains(kd.getType())) {

				final String val = XMLIndenter.indent(e.getValue().build());

				if ((xpaths != null) && !xpaths.isEmpty() && (xpaths.get(kd.getType()) != null)) {

					final Document doc = r.read(new StringReader(val));

					log.debug("\n" + e.getKey());

					for (final String xpath : xpaths.get(kd.getType())) {

						log.debug(doc.valueOf(xpath));

					}

				} else {

					log.info(val);

				}

			}

		}

	}

	private void printNoIndent(final Map<String, XmlRecordFactory> builders) {

		for (final Entry<String, XmlRecordFactory> e : builders.entrySet()) {

			if (e.getValue().isValid()) {

				log.debug(e.getValue().build());

			} else {

				log.debug("invalid builder: " + e.getKey());

			}

		}

	}

	private void ensureBuilder(final Map<String, XmlRecordFactory> builders, final String rowKey) throws Exception {

		if (!builders.containsKey(rowKey)) {

			builders.put(rowKey, newBuilder());

		}

	}

	private XmlRecordFactory newBuilder() throws TransformerConfigurationException, TransformerFactoryConfigurationError, DocumentException, IOException {

		return new XmlRecordFactory(entityConfigTable, IndexConfigTest.loadContextFromCp(), OntologyLoader.loadOntologiesFromCp(),

				XmlRecordFactoryTest.SCHEMA_LOCATION, true, true, false);

	}

}