D-Net

package eu.dnetlib.data.mapreduce.hbase.index;

import java.io.IOException;

import java.util.List;

import java.util.Map.Entry;

import org.apache.commons.codec.binary.Base64;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrServer;

import org.apache.solr.client.solrj.response.SolrPingResponse;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mapreduce.JobParams;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory;

import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;

import eu.dnetlib.miscutils.datetime.HumanTime;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

public class IndexFeedMapper extends Mapper<Text, Text, Text, Text> {

	private InputDocumentFactory documentFactory;

	private CloudSolrServer solrServer;

	private String version;

	private String dsId;

	private int shutdownWaitTime = 10000;

	private int bufferFlushThreshold = 100;

	private ApplyXslt dmfToRecord;

	private List<SolrInputDocument> buffer;

	private int backoffTimeMs = 5000;

	private boolean simulation = false;

	@Override

	protected void setup(final Context context) throws IOException, InterruptedException {

		logConfiguration(context.getConfiguration());

		dsId = context.getConfiguration().get(JobParams.INDEX_DSID);

		shutdownWaitTime = Integer.parseInt(context.getConfiguration().get(JobParams.INDEX_SHUTDOWN_WAIT));

		bufferFlushThreshold = Integer.parseInt(context.getConfiguration().get(JobParams.INDEX_BUFFER_FLUSH_TRESHOLD));

		documentFactory = new StreamingInputDocumentFactory();

		version = InputDocumentFactory.getParsedDateField(context.getConfiguration().get(JobParams.INDEX_FEED_TIME));

		buffer = Lists.newArrayList();

		simulation = Boolean.parseBoolean(context.getConfiguration().get(JobParams.INDEX_FEED_SIMULATION_MODE));

		final String xslt = new String(Base64.decodeBase64(context.getConfiguration().get(JobParams.INDEX_XSLT)));

		System.out.println("got xslt: \n" + xslt);

		System.out.println("got version: " + version);

		System.out.println("simulation: " + simulation);

		System.out.println("buffer size: " + bufferFlushThreshold);

		dmfToRecord = new ApplyXslt(xslt);

		final String baseURL = context.getConfiguration().get(JobParams.INDEX_SOLR_URL);

		System.out.println("solr server baseURL: " + baseURL);

		final String collection = context.getConfiguration().get(JobParams.INDEX_SOLR_COLLECTION);

		System.out.println("solr server collection: " + collection);

		while (true) {

			try {

				System.out.println("initializing solr server...");

				solrServer = new CloudSolrServer(baseURL);

				solrServer.connect();

				solrServer.setParallelUpdates(true);

				solrServer.setDefaultCollection(collection);

				final SolrPingResponse rsp = solrServer.ping();

				if (rsp.getStatus() != 0) throw new SolrServerException("bad init status: " + rsp.getStatus());

				else {

					break;

				}

			} catch (final Throwable e) {

				if (solrServer != null) {

					solrServer.shutdown();

				}

				context.getCounter("index init", e.getMessage()).increment(1);

				System.out.println(String.format("failed to init solr client wait %dms", backoffTimeMs));

				Thread.sleep(backoffTimeMs);

			}

		}

	}

	@Override

	protected void map(final Text key, final Text value, final Context context) throws IOException, InterruptedException {

		String indexRecord = "";

		SolrInputDocument doc = null;

		try {

			indexRecord = dmfToRecord.evaluate(value.toString());

			doc = documentFactory.parseDocument(version, indexRecord, dsId, "dnetResult");

			if ((doc == null) || doc.isEmpty()) throw new EmptySolrDocumentException();

		} catch (final Throwable e) {

			context.getCounter("index feed", "skipped records").increment(1);

			handleError(key, value, context, indexRecord, doc, e);

			return;

		}

		while (true) {

			try {

				addDocument(context, doc);

				return;

			} catch (final Throwable e) {

				context.getCounter("index feed", "retries").increment(1);

				handleError(key, value, context, indexRecord, doc, e);

				System.out.println(String.format("failed to feed documents, waiting %dms", backoffTimeMs));

				Thread.sleep(backoffTimeMs);

			}

		}

	}

	private void addDocument(final Context context, final SolrInputDocument doc) throws SolrServerException, IOException, EmptySolrDocumentException {

		buffer.add(doc);

		if (buffer.size() >= bufferFlushThreshold) {

			doAdd(buffer, context);

		}

	}

	private void doAdd(final List<SolrInputDocument> buffer, final Context context) throws SolrServerException, IOException {

		if (!simulation) {

			final long start = System.currentTimeMillis();

			final UpdateResponse rsp = solrServer.add(buffer);

			final long stop = System.currentTimeMillis() - start;

			System.out.println("feed time for " + buffer.size() + " records : " + HumanTime.exactly(stop) + "\n");

			final int status = rsp.getStatus();

			context.getCounter("index feed", "status code: " + status).increment(buffer.size());

			if (status != 0) throw new SolrServerException("bad status: " + status);

			for (final SolrInputDocument doc : buffer) {

				context.getCounter("index entity", getEntityType(doc)).increment(1);

			}

		}

		buffer.clear();

	}

	@Override

	protected void cleanup(final Context context) throws IOException, InterruptedException {

		super.cleanup(context);

		try {

			if (!buffer.isEmpty()) {

				doAdd(buffer, context);

			}

			System.out.println("\nwaiting " + shutdownWaitTime + "ms before shutdown");

			Thread.sleep(shutdownWaitTime);

			solrServer.shutdown();

		} catch (final SolrServerException e) {

			System.err.println("couldn't shutdown server " + e.getMessage());

		}

	}

	private void handleError(final Text key, final Text value, final Context context, final String indexRecord, final SolrInputDocument doc, final Throwable e)

			throws IOException, InterruptedException {

		context.getCounter("index feed", e.getClass().getName()).increment(1);

		context.write(key, printRottenRecord(context.getTaskAttemptID().toString(), value, indexRecord, doc));

		// e.printStackTrace(System.err);

	}

	private Text printRottenRecord(final String taskid, final Text value, final String indexRecord, final SolrInputDocument doc) {

		return new Text("\n**********************************\n" + "task: " + taskid + "\n"

				+ check("original", value.toString() + check("indexRecord", indexRecord) + check("solrDoc", doc)));

	}

	private String check(final String label, final Object value) {

		if ((value != null) && !value.toString().isEmpty()) return "\n " + label + ":\n" + value + "\n";

		return "\n";

	}

	private void logConfiguration(final Configuration conf) {

		System.out.println("job configutation #################");

		for (final Entry<String, String> e : conf) {

			System.out.println("'" + e.getKey() + "' : '" + e.getValue() + "'");

		}

		System.out.println("end of job configutation #################\n\n");

	}

	private String getEntityType(final SolrInputDocument doc) {

		if (!doc.containsKey("oaftype")) return "unknown";

		final Type type = Type.valueOf(doc.getFieldValue("oaftype").toString());

		switch (type) {

		case result:

			if (!doc.containsKey("resulttypeid")) return "result";

			return doc.getFieldValue("resulttypeid").toString();

		default:

			return type.toString();

		}

	}

}