D-Net

package eu.dnetlib.openaire.blacklist;

import java.util.List;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import junit.framework.Assert;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.runners.MockitoJUnitRunner;

@RunWith(MockitoJUnitRunner.class)

public class OpenaireIdResolverTest {

	private static final Log log = LogFactory.getLog(OpenaireIdResolverTest.class);

	private final String id = "dedup_wf_001::00066fdc65f1e21677c69d976a8e852d";

	private OpenaireIdResolver resolver;

	@Mock

	private ISLookUpService isLookupService;

	@Mock

	private UniqueServiceLocator serviceLocator;

	@Before

	public void setUp() throws Exception {

		resolver = new OpenaireIdResolver();

		Mockito.when(serviceLocator.getService(ISLookUpService.class)).thenReturn(isLookupService);

		Mockito.when(isLookupService.getResourceProfile(Mockito.startsWith("for $x in collection('/db/DRIVER/ServiceResources/IndexServiceResourceType')")))

				.thenReturn("beta.solr.openaire.eu:9983");

		Mockito.when(isLookupService.getResourceProfile(Mockito.startsWith("for $x in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')")))

				.thenReturn("DMF");

		resolver.setServiceLocator(serviceLocator);

	}

	@Ignore

	@Test

	public void testFindOriginalIds() throws Exception {

		log.debug("Testing for " + id);

		List<String> originals = resolver.findOriginalIds(id);

		Assert.assertTrue(originals.contains("od_______165::deb6d865011c09f228af41a30e0ff1a4"));

		Assert.assertTrue(originals.contains("od_______165::00066fdc65f1e21677c69d976a8e852d"));

		Assert.assertTrue(originals.size() == 2);

	}

}

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ROW><FIELD name=\"iis_module\">iis::document_referencedProjects</FIELD><FIELD name=\"ticket_id\">1245</FIELD><FIELD name=\"last_update_time\">2015-09-18 12:39:11.781532</FIELD><FIELD name=\"status\">accepted</FIELD><FIELD name=\"userid\">unknown</FIELD><FIELD name=\"iis_module_profile\" isNull=\"true\"/><FIELD name=\"original_target_objects\" isNull=\"true\"/><FIELD name=\"relationship\">resultProject_outcome_isProducedBy</FIELD><FIELD name=\"id\">1</FIELD><FIELD name=\"provenance\">iis</FIELD><FIELD name=\"iis_status\">unsolved</FIELD><FIELD name=\"original_source_objects\" isNull=\"true\"/><FIELD name=\"creation_time\">2015-09-18 12:39:11.781532</FIELD><FIELD name=\"target_object\">corda_______::fda6a1c19dac47508b6fb619518328c7</FIELD><FIELD name=\"note\" isNull=\"true\"/><FIELD name=\"source_object\">od_______908::a47e1c3ede9a21ee5278a2e5c338d69b</FIELD><FIELD name=\"source_type\">result</FIELD><FIELD name=\"target_type\">project</FIELD></ROW>");

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ROW><FIELD name=\"iis_module\">iis::document_referencedProjects</FIELD><FIELD name=\"ticket_id\">1245</FIELD><FIELD name=\"last_update_time\">2015-09-18 12:39:11.789433</FIELD><FIELD name=\"status\">accepted</FIELD><FIELD name=\"userid\">unknown</FIELD><FIELD name=\"iis_module_profile\" isNull=\"true\"/><FIELD name=\"original_target_objects\" isNull=\"true\"/><FIELD name=\"relationship\">resultProject_outcome_isProducedBy</FIELD><FIELD name=\"id\">2</FIELD><FIELD name=\"provenance\">iis</FIELD><FIELD name=\"iis_status\">unsolved</FIELD><FIELD name=\"original_source_objects\" isNull=\"true\"/><FIELD name=\"creation_time\">2015-09-18 12:39:11.789433</FIELD><FIELD name=\"target_object\">corda_______::189ff31d637eaaeaf4d3584dc490b1cf</FIELD><FIELD name=\"note\" isNull=\"true\"/><FIELD name=\"source_object\">od_______908::a47e1c3ede9a21ee5278a2e5c338d69b</FIELD><FIELD name=\"source_type\">result</FIELD><FIELD name=\"target_type\">project</FIELD></ROW>");

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ROW><FIELD name=\"iis_module\">iis::document_referencedProjects</FIELD><FIELD name=\"ticket_id\">1297</FIELD><FIELD name=\"last_update_time\">2015-09-18 12:39:11.854513</FIELD><FIELD name=\"status\">accepted</FIELD><FIELD name=\"userid\">unknown</FIELD><FIELD name=\"iis_module_profile\" isNull=\"true\"/><FIELD name=\"original_target_objects\" isNull=\"true\"/><FIELD name=\"relationship\">resultProject_outcome_isProducedBy</FIELD><FIELD name=\"id\">12</FIELD><FIELD name=\"provenance\">iis</FIELD><FIELD name=\"iis_status\">unsolved</FIELD><FIELD name=\"original_source_objects\" isNull=\"true\"/><FIELD name=\"creation_time\">2015-09-18 12:39:11.854513</FIELD><FIELD name=\"target_object\">corda_______::35695c955c51f0bb39482ce5477047c7</FIELD><FIELD name=\"note\" isNull=\"true\"/><FIELD name=\"source_object\">webcrawl____::ae0ae3ea5641ac24fa148ac38397dfcd</FIELD><FIELD name=\"source_type\">result</FIELD><FIELD name=\"target_type\">project</FIELD></ROW>");

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ROW><FIELD name=\"iis_module\">iis::document_referencedProjects</FIELD><FIELD name=\"ticket_id\">1297</FIELD><FIELD name=\"last_update_time\">2015-09-18 12:39:11.895653</FIELD><FIELD name=\"status\">accepted</FIELD><FIELD name=\"userid\">unknown</FIELD><FIELD name=\"iis_module_profile\" isNull=\"true\"/><FIELD name=\"original_target_objects\" isNull=\"true\"/><FIELD name=\"relationship\">resultProject_outcome_isProducedBy</FIELD><FIELD name=\"id\">17</FIELD><FIELD name=\"provenance\">iis</FIELD><FIELD name=\"iis_status\">unsolved</FIELD><FIELD name=\"original_source_objects\" isNull=\"true\"/><FIELD name=\"creation_time\">2015-09-18 12:39:11.895653</FIELD><FIELD name=\"target_object\">corda_______::35695c955c51f0bb39482ce5477047c7</FIELD><FIELD name=\"note\" isNull=\"true\"/><FIELD name=\"source_object\">webcrawl____::6ce2789da52c6036501533ed5613d71a</FIELD><FIELD name=\"source_type\">result</FIELD><FIELD name=\"target_type\">project</FIELD></ROW>");

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ROW><FIELD name=\"iis_module\">iis::document_referencedProjects</FIELD><FIELD name=\"ticket_id\">1360</FIELD><FIELD name=\"last_update_time\">2015-09-18 12:39:11.955417</FIELD><FIELD name=\"status\">accepted</FIELD><FIELD name=\"userid\">unknown</FIELD><FIELD name=\"iis_module_profile\" isNull=\"true\"/><FIELD name=\"original_target_objects\" isNull=\"true\"/><FIELD name=\"relationship\">resultProject_outcome_isProducedBy</FIELD><FIELD name=\"id\">23</FIELD><FIELD name=\"provenance\">iis</FIELD><FIELD name=\"iis_status\">unsolved</FIELD><FIELD name=\"original_source_objects\" isNull=\"true\"/><FIELD name=\"creation_time\">2015-09-18 12:39:11.955417</FIELD><FIELD name=\"target_object\">corda_______::800e636c0a24ac3767701a789a96ba89</FIELD><FIELD name=\"note\" isNull=\"true\"/><FIELD name=\"source_object\">od______2367::eb76f733498920166d4ce77ec37d487c</FIELD><FIELD name=\"source_type\">result</FIELD><FIELD name=\"target_type\">project</FIELD></ROW>");

### Root Level ###

log4j.rootLogger=WARN, CONSOLE

### Configuration for the CONSOLE appender ###

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c - %m%n

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

### Application Level ###

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.openaire.blacklist=DEBUG

package eu.dnetlib.openaire.blacklist;

import java.io.StringReader;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.client.solrj.impl.CloudSolrServer;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocument;

import org.apache.solr.common.SolrDocumentList;

import org.dom4j.Attribute;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * The goal of this task is to return the original identifiers of objects merged in a representative object by deduplication.

 * Created by alessia on 09/02/16.

 */

public class OpenaireIdResolver {

	public final static String SOLR_COLLECTION_POSTFIX = "-index-openaire";

	private static final Log log = LogFactory.getLog(OpenaireIdResolver.class);

	private final static String RESULT_FIELD = "__result";

	private final static String XPATH_TO_MERGED = "//*[local-name()='entity']/*//children/result/@objidentifier";

	private final SAXReader saxReader = new SAXReader();

	@Autowired

	private UniqueServiceLocator serviceLocator;

	public List<String> resolveIdentifier(final String id) {

		if (StringUtils.isBlank(id)) return Lists.newArrayList();

		else {

			return findOriginalIds(id);

		}

	}

	protected List<String> findOriginalIds(final String id) {

		CloudSolrServer solrCore = null;

		SolrQuery q = new SolrQuery("objidentifier:\"" + id + "\"");

		QueryResponse response = null;

		try {

			solrCore = new CloudSolrServer(getIndexEndpoint());

			solrCore.setDefaultCollection(getPublicIndexCollection() + SOLR_COLLECTION_POSTFIX);

			response = solrCore.query(q);

			SolrDocumentList results = response.getResults();

			if (results.isEmpty()) {

				log.debug("Query " + q + " returned 0 documents");

				return Lists.newArrayList();

			}

			//my results contain the document with the given identifier

			SolrDocument solrDoc = results.get(0);

			return extractMergedIdentifiers(solrDoc);

		} catch (Exception e) {

			log.error(e);

			throw new RuntimeException("Can't get original ids for " + id + "\n " + e);

		} finally {

			solrCore.shutdown();

		}

	}

	protected List<String> extractMergedIdentifiers(final SolrDocument doc) throws DocumentException {

		List<String> originals = Lists.newArrayList();

		String xmlRecord = (String) doc.getFirstValue(RESULT_FIELD);

		Document xmlDoc = this.saxReader.read(new StringReader(xmlRecord));

		return Lists.newArrayList(Iterables.transform(xmlDoc.selectNodes(XPATH_TO_MERGED), new Function<Attribute, String>() {

			@Override

			public String apply(final Attribute a) {

				return a.getStringValue();

			}

		}));

	}

	protected String getIndexEndpoint() throws ISLookUpException {

		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfile(

				"for $x in collection('/db/DRIVER/ServiceResources/IndexServiceResourceType') return $x//PROTOCOL[./@name='solr']/@address/string()");

	}

	protected String getPublicIndexCollection() throws ISLookUpException {

		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfile(

				"for $x in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType') return $x[.//PROPERTY[@key='infrastructure']/@value='public']//PROPERTY[@key='mdformat']/@value/string()");

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

	@Autowired

	private OpenaireIdResolver openaireIdResolver;

	private List<String> getOriginalIds(String id, String entityType) {

		List<String> originalIds = Lists.newArrayList();

		//We need to find original ids only for entities that are deduplicated: result, organization, person.

		if (entityType.equals("result") || entityType.equals("organization") || entityType.equals("person")) {

			originalIds = openaireIdResolver.resolveIdentifier(id);

		}

		return originalIds;

	}

	    List<String> sourceIds = getOriginalIds(entry.getSourceObject(), entry.getSourceType());

	    List<String> targetIds = getOriginalIds(entry.getTargetObject(), entry.getTargetType());

	    entry.setOriginalSourceObjects(sourceIds);

	    entry.setOriginalTargetObjects(targetIds);

	    String addQuery = String.format(

			    "INSERT INTO blacklist(userid, relationship, provenance, iis_module, iis_status, status, source_object, source_type, target_object, target_type, "

					    + "ticket_id, original_source_objects, original_target_objects, note) "

					    + " VALUES('%1$s', '%2$s', '%3$s', '%4$s', '%5$s', '%6$s', '%7$s', '%8$s', '%9$s', '%10$s', '%11$s', '%12$s', '%13$s', '%14$s') ",

			    entry.getUser(),

			    entry.getRelationship(),

			    entry.getProvenance(),

			    entry.getIisModule(),

			    entry.getIisStatus(),

			    entry.getStatus(),

			    entry.getSourceObject(),

			    entry.getSourceType(),

			    entry.getTargetObject(),

			    entry.getTargetType(),

			    entry.getTicketId(),

			    joinCollectionForSQL(sourceIds.isEmpty() ?

					    Lists.newArrayList(entry.getSourceObject()) :

					    sourceIds),

			    joinCollectionForSQL(targetIds.isEmpty() ?

					    Lists.newArrayList(entry.getTargetObject()) :

					    targetIds),

			    entry.getNote());

	    log.debug("Adding new blacklist entry");

		                blacklistDatabaseName, addQuery);

	    List<String> sourceIds = getOriginalIds(entry.getSourceObject(), entry.getSourceType());

	    List<String> targetIds = getOriginalIds(entry.getTargetObject(), entry.getTargetType());

	    entry.setOriginalSourceObjects(sourceIds);

	    entry.setOriginalTargetObjects(targetIds);

	    DatabaseService dbService = serviceLocator.getService(DatabaseService.class);

	    String editQuery = String.format(

			    "UPDATE blacklist SET userid='%s', relationship='%s', provenance='%s', iis_module='%s', iis_status='%s', status='%s', source_object='%s', source_type='%s', "

					    + "target_object='%s', target_type='%s', ticket_id='%s', original_source_objects='%s', original_target_objects='%s', note='%s' WHERE id='%s' ",

			    entry.getUser(),

			    entry.getRelationship(),

			    entry.getProvenance(),

			    entry.getIisModule(),

			    entry.getIisStatus(),

			    entry.getStatus(),

			    entry.getSourceObject(),

			    entry.getSourceType(),

			    entry.getTargetObject(),

			    entry.getTargetType(),

			    entry.getTicketId(),

			    joinCollectionForSQL(sourceIds.isEmpty() ?

					    Lists.newArrayList(entry.getSourceObject()) :

					    sourceIds),

			    joinCollectionForSQL(targetIds.isEmpty() ?

					    Lists.newArrayList(entry.getTargetObject()) :

					    targetIds),

			    entry.getNote(),

			    entry.getId());

	    log.debug("Editing blacklist entry: " + entry.getId());

		                blacklistDatabaseName, editQuery);

	    log.debug("Deleting entry " + entryId + " from blacklist");

	    return this

			    .safeUpdateSql(

					    dbService,

					    blacklistDatabaseName,

					    String.format(

							    "DELETE FROM blacklist WHERE id='%s' ", entryId));

	                be.setSourceType(doc.selectSingleNode("//FIELD[./@name='source_type']").getText());

	                be.setTargetType(doc.selectSingleNode("//FIELD[./@name='target_type']").getText());

		UNSOLVED, SOLVED

	    PENDING, ACCEPTED, REFUSED, DELETED

    private String note;

    private String ticketId;

     * Provenance of the relationship. 'iis' when the relationship is inferred. 'crosswalk' if it is aggregated. 'claim' if it is a claim.

    private String provenance;

	/**

	 * Relationship label: 'resultProject_outcome_isProducedBy' for result-project relationships.

	 */

	private String relationship;

     * The inference module that generated the relationship. 'iis::document_referencedProjects' for result-project relationships. Blank if provenance is not 'iis'.

    private String iisModule = "";

    private IIS_STATUS iisStatus = null;

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns="http://www.springframework.org/schema/beans"

	<bean id="openaireIdResolver" class="eu.dnetlib.openaire.blacklist.OpenaireIdResolver"/>

		    <select class="form-control" id="sourceTypeSelect" ng-model="entry.sourceType">

			<input class="form-control" id="sourceObject" ng-model="entry.sourceObject"

			<select class="form-control" id="relationType" ng-model="entry.relationship">

		    <select class="form-control" required id="targetTypeSelect" ng-model="entry.targetType">

			<input class="form-control" id="targetObject" ng-model="entry.targetObject"

	<div class="form-group">

		<label for="provenanceSelect">Provenance</label>

		<div><select class="form-control" id="provenanceSelect" ng-model="entry.provenance">

			<option>iis</option>

			<option>crosswalk</option>

			<option>claim</option>

			<option>unknown</option>

		</select></div>

	</div>

	<!--

			       type="text" placeholder="E.g. iis, crosswalk, claim"/>

	-->

	    <div><textarea class="form-control" rows="3" ng-model="entry.note" id="note"></textarea>

<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <version>0.0.6-PROD-SNAPSHOT</version>

	    <dependency>

		    <groupId>org.mockito</groupId>

		    <artifactId>mockito-core</artifactId>

		    <version>1.9.5</version>

        </dependency>