D-Net

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertNull;

import static org.mockito.Matchers.anyString;

import static org.mockito.Mockito.times;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import java.io.StringReader;

import java.util.List;

import org.dom4j.Document;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.runners.MockitoJUnit44Runner;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

@RunWith(MockitoJUnit44Runner.class)

public class VocabularyRuleTest {

	private static final String VOCABULARY_NAME_1 = "TEST VOCABULARY 1";

	private static final String VOCABULARY_NAME_2 = "TEST VOCABULARY 2";

	private static final List<String> VOCABULARY = Lists.newArrayList("XXXX|-:-|AAAA", "YYYY|-:-|AAAA", "ZZZZ|-:-|AAAA");

	/**

	 * Class Under Test

	 */

	private VocabularyRule rule;

	@Mock

	private ServiceLocator<ISLookUpService> lookupLocator;

	@Mock

	private ISLookUpService lookup;

	@Before

	public void setUp() throws Exception {

		when(lookupLocator.getService()).thenReturn(lookup);

		when(lookup.quickSearchProfile(anyString())).thenReturn(VOCABULARY);

		rule = new VocabularyRule(Sets.newHashSet(VOCABULARY_NAME_1, VOCABULARY_NAME_2), lookupLocator);

	}

	@Test

	public void testSetup() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		execute("<a><b>XXXX</b></a>");

		verify(lookup, times(2)).quickSearchProfile(anyString());

		assertEquals(VOCABULARY.size(), rule.getVocabularyTerms().size());

	}

	@Test

	public void testApplyXpathRule() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_2() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_3() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_attr() throws Exception {

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b value='XXXX' /></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_with_spaces() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>  XXXX  </b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("  XXXX  "));

	}

	@Test

	public void testApplyXpathRule_case() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>Xxxx</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("Xxxx"));

	}

	private Document execute(final String xml) throws Exception {

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(xml));

		System.out.println("BEFORE:\n" + doc.asXML() + "\n");

		rule.applyXpathRule(doc);

		System.out.println("AFTER:\n" + doc.asXML() + "\n");

		System.out.println("-----------------------------\n");

		return doc;

	}

}

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import java.util.HashMap;

import java.util.Map;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnit44Runner;

import com.google.common.collect.Lists;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

@RunWith(MockitoJUnit44Runner.class)

public class XMLCleaningRuleTest {

	private static final String INPUT_VALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>HELLO</a>" + "	</metadata>" + "</record>";

	private static final String INPUT_INVALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>GOOD BYE</a>" + "	</metadata>" + "</record>";

	/**

	 * Class under test.

	 */

	private CleaningRule xmlRule;

	private XPATHCleaningRule mockXpathRule = new XPATHCleaningRule() {

		@Override

		protected Map<String, String> verifyValue(final String value) throws CleanerException {

			if (value.equals("CIAO")) { return null; }

			Map<String, String> err = new HashMap<String, String>();

			err.put("term", value);

			return err;

		}

		@Override

		protected String calculateNewValue(final String oldValue) throws CleanerException {

			if (oldValue.equals("HELLO")) { return "CIAO"; }

			return oldValue;

		}

	};;

	@Before

	public void setUp() throws Exception {

		xmlRule = new CleaningRule();

		mockXpathRule.setStrict(true);

		mockXpathRule.setXpath("//a");

		xmlRule.setXpathRules(Lists.newArrayList(mockXpathRule));

	}

	@Test

	public void testEvaluate_valid() {

		String s = xmlRule.evaluate(INPUT_VALID);

		assertTrue(s.contains("CIAO"));

		assertFalse(s.contains("invalid"));

	}

	@Test

	public void testEvaluate_invalid() {

		String s = xmlRule.evaluate(INPUT_INVALID);

		System.out.println(s);

		assertFalse(s.contains("CIAO"));

		assertTrue(s.contains("invalid"));

	}

}

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.runners.MockitoJUnit44Runner;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.test.utils.EPRTestUtil;

@RunWith(MockitoJUnit44Runner.class)

public class CleanerServiceImplTest {

	/**

	 * Class under test.

	 */

	private CleanerServiceImpl service;

	@Mock

	private CleaningRuleFactory cleaningRuleFactory;

	@Mock

	private MappedResultSetFactory mappedResultSetFactory;

	@Mock

	private CleaningRule cleaningRule;

	private W3CEndpointReference epr_IN = EPRTestUtil.getTestEpr("http://1");

	private W3CEndpointReference epr_OUT = EPRTestUtil.getTestEpr("http://2");

	private static final String RULE_ID = "RULE_01";

	@Before

	public void setUp() throws Exception {

		when(cleaningRuleFactory.obtainCleaningRule(RULE_ID)).thenReturn(cleaningRule);

		when(mappedResultSetFactory.createMappedResultSet(epr_IN, cleaningRule)).thenReturn(epr_OUT);

		service = new CleanerServiceImpl();

		service.setCleaningRuleFactory(cleaningRuleFactory);

		service.setMappedResultSetFactory(mappedResultSetFactory);

	}

	@Test

	public void testClean() throws CleanerException {

		W3CEndpointReference epr = service.clean(epr_IN, RULE_ID);

		assertNotNull(epr);

		assertEquals(epr_OUT, epr);

		verify(cleaningRuleFactory).obtainCleaningRule(RULE_ID);

	}

	@Test(expected = CleanerException.class)

	public void testClean_null_1() throws CleanerException {

		service.clean(epr_IN, null);

	}

	@Test(expected = CleanerException.class)

	public void testClean_null_2() throws CleanerException {

		service.clean(null, RULE_ID);

	}

}

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import java.io.StringReader;

import java.util.List;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnit44Runner;

@RunWith(MockitoJUnit44Runner.class)

public class GroovyRuleTest {

	@Test

	public void testApplyXpathRule_simple_constant() throws Exception {

		final GroovyRule rule = new GroovyRule("'YYYY'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>XXXX</b></a>");

		assertEquals("YYYY", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_simple_regex() throws Exception {

		final GroovyRule rule = new GroovyRule("(input =~ /X/).replaceAll('Y')");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>aXaXa</b></a>");

		assertEquals("aYaYa", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_simple_upper() throws Exception {

		final GroovyRule rule = new GroovyRule("input.toUpperCase()");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>xyz</b></a>");

		assertEquals("XYZ", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_multi() throws Exception {

		final GroovyRule rule = new GroovyRule("'Y'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>X</b><b>X</b><b>X</b></a>");

		List<?> list = doc.selectNodes(xpath);

		assertEquals(3, list.size());

		for (Object o : list) {

			assertEquals("Y", ((Node) o).getText());

		}

	}

	@Test

	public void testApplyXpathRule_singleAttr() throws Exception {

		final GroovyRule rule = new GroovyRule("'BBBB'");

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b value='AAAA'>XXXX</b></a>");

		assertEquals("BBBB", doc.valueOf(xpath));

		assertEquals("XXXX", doc.valueOf("/a/b"));

	}

	@Test

	public void testApplyXpathRule_multiAttr() throws Exception {

		final GroovyRule rule = new GroovyRule("'B'");

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b value='a' /><b value='b' /><b value='c' /></a>");

		final List<?> list = doc.selectNodes(xpath);

		assertEquals(3, list.size());

		for (Object o : list) {

			assertEquals("B", ((Node) o).getText());

		}

	}

	@Test

	public void testApplyXpathRule_complex() throws Exception {

		final GroovyRule rule = new GroovyRule("'B'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>X<c>C</c></b></a>");

		assertTrue(doc.valueOf(xpath).contains("B"));

		assertEquals("C", doc.valueOf("/a/b/c"));

	}

	private Document execute(final GroovyRule rule, final String xml) throws Exception {

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(xml));

		System.out.println("BEFORE:\n" + doc.asXML() + "\n");

		rule.applyXpathRule(doc);

		System.out.println("AFTER:\n" + doc.asXML() + "\n");

		System.out.println("-----------------------------\n");

		return doc;

	}

}

package eu.dnetlib.data.utility.cleaner;

import java.util.HashMap;

import java.util.Map;

import java.util.Set;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Joiner;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

/**

 * @author michele

 * 

 *         Vocabulary rules must be declared in a CleanerDS profile, for each vocabulary must be present the relative VocabularyDS profile:

 * 

 *         <RULE xpath="..." vocabularies="VOC1" /> <RULE xpath="..." vocabularies="VOC1, VOC2, VOC3" />

 */

public class VocabularyRule extends XPATHCleaningRule {

	private Set<String> vocabularies;

	private static final Log log = LogFactory.getLog(VocabularyRule.class); // NOPMD by marko on 11/24/08 5:02 PM

	private Map<String, String> synonyms = Maps.newHashMap();

	private Set<String> validTerms = Sets.newHashSet();

	public VocabularyRule(final Set<String> vocabularies, final ServiceLocator<ISLookUpService> lookupLocator) throws CleanerException {

		this.vocabularies = vocabularies;

		loadSynonymsAndTerms(lookupLocator);

	}

	@Override

	protected String calculateNewValue(final String oldValue) throws CleanerException {

		log.debug("calculating new value for: " + oldValue);

		if (synonyms.isEmpty()) {

			log.warn("Vocabulary terms is void, vocabularies: " + this.vocabularies);

		}

		String newValue = null;

		if (synonyms.containsKey(oldValue.toLowerCase())) {

			newValue = synonyms.get(oldValue.toLowerCase());

		}

		if (newValue == null) {

			log.debug("Synonym " + oldValue + " not found in vocabulary");

			return oldValue;

		}

		return newValue;

	}

	private void loadSynonymsAndTerms(final ServiceLocator<ISLookUpService> lookupLocator) throws CleanerException {

		for (final String vocabulary : vocabularies) {

			try {

				final String query = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')"

						+ "//RESOURCE_PROFILE[.//VOCABULARY_NAME='" + vocabulary + "']//TERM return "

						+ "( concat($x/@code,'|-:-|', $x/@code), concat($x/@english_name,'|-:-|', $x/@code), concat($x/@native_name,'|-:-|', $x/@code), "

						+ "for $y in $x//SYNONYM return concat($y/@term,'|-:-|', $x/@code) )";

				for (final String s : lookupLocator.getService().quickSearchProfile(query)) {

					log.debug("SYNONYM : " + s);

					final String[] arr = s.split("\\|-:-\\|");

					if ((arr[0] == null) || arr[0].isEmpty()) {

						continue;

					}

					synonyms.put(arr[0].toLowerCase(), arr[1]);

					validTerms.add(arr[1].toLowerCase());

				}

				log.info("VOCABULARY " + vocabulary.trim() + " - terms size " + synonyms.size());

			} catch (final Exception e) {

				throw new CleanerException("Error obtaining vocabulary " + vocabulary, e);

			}

		}

	}

	@Override

	protected Map<String, String> verifyValue(final String value) throws CleanerException {

		if (synonyms.isEmpty()) {

			log.warn("Vocabulary terms is void, vocabularies: " + this.vocabularies);

		}

		if (validTerms.contains(value.toLowerCase())) { return null; }

		final Map<String, String> error = new HashMap<String, String>();

		error.put("term", value);

		error.put("vocabularies", this.vocabularies.toString().replaceAll("\\[", "").replaceAll("\\]", ""));

		error.put("xpath", this.getXpath());

		return error;

	}

	public Map<String, String> getVocabularyTerms() {

		return synonyms;

	}

	@Override

	public String toString() {

		return "VOCABULARIES: [" + Joiner.on(", ").join(vocabularies) + "]";

	}

}

package eu.dnetlib.data.utility.cleaner;

import java.io.StringReader;

import java.util.HashSet;

import java.util.List;

import java.util.Set;

import org.dom4j.Document;

import org.dom4j.Element;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

public class CleaningRuleFactory {

	private ServiceLocator<ISLookUpService> lookupLocator;

	public CleaningRule obtainCleaningRule(final String ruleId) throws CleanerException {

		try {

			final String prof = lookupLocator.getService().getResourceProfileByQuery(

					"/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + ruleId + "' or .//CLEANER_NAME='" + ruleId + "']//CONFIGURATION");

			final SAXReader reader = new SAXReader();

			final Document doc = reader.read(new StringReader(prof));

			final CleaningRule rule = new CleaningRule();

			for (Object o : doc.selectNodes("//RULE")) {

				final Element node = (Element) o;

				final String xpath = node.valueOf("@xpath");

				final String vocabularies = node.valueOf("@vocabularies");

				final String groovyRule = node.valueOf("@groovy");

				final String strict = node.valueOf("@strict");

				final XPATHCleaningRule xpathRule;

				if ((vocabularies != null) && (vocabularies.length() > 0)) {

					final Set<String> list = Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(vocabularies));

					xpathRule = new VocabularyRule(list, lookupLocator);

				} else {

					xpathRule = new GroovyRule(groovyRule);

				}

				xpathRule.setXpath(xpath);

				xpathRule.setStrict("true".equals(strict));

				rule.getXpathRules().add(xpathRule);

			}

			return rule;

		} catch (Exception e) {

			throw new CleanerException("Error obtaing cleaner rule " + ruleId, e);

		}

	}

	public List<String> getRuleIds() throws CleanerException {

		try {

			final HashSet<String> response = new HashSet<String>();

			final List<String> list = lookupLocator.getService().quickSearchProfile("//CLEANER_NAME");

			if (list != null) {

				response.addAll(list);

			}

			return Lists.newArrayList(response);

		} catch (ISLookUpException e) {

			throw new CleanerException("Error obtaining IDs of cleaner DSs", e);

		}

	}

	@Required

	public void setLookupLocator(final ServiceLocator<ISLookUpService> lookupLocator) {

		this.lookupLocator = lookupLocator;

	}

	public ServiceLocator<ISLookUpService> getLookupLocator() {

		return lookupLocator;

	}

}

package eu.dnetlib.data.utility.cleaner.inspector;

import java.util.List;

import javax.annotation.Resource;

import org.springframework.stereotype.Controller;

import org.springframework.ui.Model;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestParam;

import com.google.common.collect.Lists;

import eu.dnetlib.data.utility.cleaner.CleaningRule;

import eu.dnetlib.data.utility.cleaner.CleaningRuleFactory;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import eu.dnetlib.enabling.inspector.AbstractInspectorController;

import eu.dnetlib.miscutils.collections.MappedCollection;

import eu.dnetlib.miscutils.functional.UnaryFunction;

@Controller

public class CleanerInspector extends AbstractInspectorController {

	@Resource

	private CleaningRuleFactory cleaningRuleFactory;

	public static class SelectOption {

		private String value;

		private boolean selected;

		public SelectOption(final String value, final boolean selected) {

			super();

			this.value = value;

			this.selected = selected;

		}

		public String getValue() {

			return value;

		}

		public void setValue(final String value) {

			this.value = value;

		}

		public boolean isSelected() {

			return selected;

		}

		public void setSelected(final boolean selected) {

			this.selected = selected;

		}

	}

	@RequestMapping(value = "/inspector/cleaner.do")

	public void cleaner(final Model model,

			@RequestParam(value = "rule", required = false) final String ruleId,

			@RequestParam(value = "dirtyText", required = false) final String dirtyText) throws CleanerException {

		List<String> rules = Lists.newArrayList(cleaningRuleFactory.getRuleIds());

		model.addAttribute("rules", selectOptions(rules, ruleId));

		if ((ruleId != null) && (dirtyText != null)) {

			CleaningRule rule = cleaningRuleFactory.obtainCleaningRule(ruleId);

			model.addAttribute("dirtyText", dirtyText);

			model.addAttribute("cleanedText", rule.evaluate(dirtyText));

		}

	}

	/**

	 * Given an list of values, return a list of SelectOption instances which have the "selected" boolean field set to true only for the

	 * element matching "current".

	 * 

	 * @param input

	 *            list of input strings

	 * @param current

	 *            current value to select

	 * @return

	 */

	private List<SelectOption> selectOptions(final List<String> input, final String current) {

		final UnaryFunction<SelectOption, String> mapper = new UnaryFunction<SelectOption, String>() {

			@Override

			public SelectOption evaluate(final String value) {

				return new SelectOption(value, value.equals(current));

			}

		};

		return Lists.newArrayList(new MappedCollection<SelectOption, String>(input, mapper));

	}

}

package eu.dnetlib.data.utility.cleaner;

import java.io.StringReader;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Element;

import org.dom4j.Namespace;

import org.dom4j.QName;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class CleaningRule implements UnaryFunction<String, String> {

	private static final Log log = LogFactory.getLog(CleaningRule.class); // NOPMD by marko on 11/24/08 5:02 PM

	private List<XPATHCleaningRule> xpathRules = new ArrayList<XPATHCleaningRule>();

	@Override

	public String evaluate(final String text) {

		try {

			final List<Map<String, String>> errors = new ArrayList<Map<String, String>>();

			final Document doc = (new SAXReader()).read(new StringReader(text));

			for (final XPATHCleaningRule r : xpathRules) {

				errors.addAll(r.applyXpathRule(doc));

			}

			if (errors.size() > 0) {

				markAsInvalid(doc, errors);

			}

			return doc.asXML();

		} catch (final Exception e) {

			log.error("Error evaluating rule", e);

		}

		return "";

	}

	private void markAsInvalid(final Document doc, final List<Map<String, String>> errors) {

		final Element element = (Element) doc.selectSingleNode("//*[local-name()='header']");

		if (element != null) {

			final Element inv = element.addElement(new QName("invalid", new Namespace("dri", "http://www.driver-repository.eu/namespace/dri")));

			for (final Map<String, String> e : errors) {

				final Element err = inv.addElement(new QName("error", new Namespace("dri", "http://www.driver-repository.eu/namespace/dri")));

				for (final Map.Entry<String, String> entry : e.entrySet()) {

					err.addAttribute(entry.getKey(), entry.getValue());

				}

			}

			inv.addAttribute("value", "true");

		}

	}

	public List<XPATHCleaningRule> getXpathRules() {

		return xpathRules;

	}

	@Required

	public void setXpathRules(final List<XPATHCleaningRule> xpathRules) {

		this.xpathRules = xpathRules;

	}

}

package eu.dnetlib.data.utility.cleaner;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerService;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

public class CleanerServiceImpl extends AbstractBaseService implements CleanerService {

	private CleaningRuleFactory cleaningRuleFactory;

	private MappedResultSetFactory mappedResultSetFactory;

	@Override

	public W3CEndpointReference clean(final W3CEndpointReference epr, final String ruleId) throws CleanerException {

		if ((ruleId == null) || (ruleId.isEmpty())) { throw new CleanerException("Invalid ruleId: id is empty"); }

		if (epr == null) { throw new CleanerException("Passed epr is empty"); }

		return mappedResultSetFactory.createMappedResultSet(epr, cleaningRuleFactory.obtainCleaningRule(ruleId));

	}

	@Required

	public MappedResultSetFactory getMappedResultSetFactory() {

		return mappedResultSetFactory;

	}

	@Required

	public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {

		this.mappedResultSetFactory = mappedResultSetFactory;

	}

	public CleaningRuleFactory getCleaningRuleFactory() {

		return cleaningRuleFactory;

	}

	@Required

	public void setCleaningRuleFactory(final CleaningRuleFactory cleaningRuleFactory) {

		this.cleaningRuleFactory = cleaningRuleFactory;

	}

}

package eu.dnetlib.data.utility.cleaner;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

import groovy.lang.Closure;

import groovy.lang.GroovyShell;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * @author michele

 * 

 *         Groovy rules must be declared in a CleanerDS profile, some examples:

 * 

 *         <RULE xpath="..." groovy="(input =~ /X/).replaceAll('Y')" /> <RULE xpath="..." groovy="'CONSTANT'" /> <RULE xpath="..."

 *         groovy="input.toUpperCase()" />

 */

public class GroovyRule extends XPATHCleaningRule {

	private static final Log log = LogFactory.getLog(GroovyRule.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String groovyRule;

	private Closure<String> closure;

	private GroovyShell groovyShell = new GroovyShell();

	@SuppressWarnings("unchecked")

	public GroovyRule(final String groovyRule) {

		this.groovyRule = groovyRule;

		this.closure = (Closure<String>) groovyShell.evaluate("{ input -> " + groovyRule + "}");

	}

	@Override

	protected String calculateNewValue(final String oldValue) throws CleanerException {

		try {

			log.info("Executing groovy closure on value " + oldValue);

			return closure.call(oldValue);

		} catch (Exception e) {

			log.error("Failed Groovy execution, groovyRule: " + groovyRule + ", input: " + oldValue, e);

			throw new CleanerException("Error executing groovy", e);

		}

	}

	@Override

	protected Map<String, String> verifyValue(final String value) throws CleanerException {

		return null;

	}

	@Override

	public String toString() {

		return "GROOVY: " + groovyRule;

	}

}

package eu.dnetlib.data.utility.cleaner;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Node;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Lists;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

public abstract class XPATHCleaningRule {

	private String xpath;

	private boolean strict = false;

	private static final Log logCleaningRules = LogFactory.getLog("VOCABULARY_RULES");

	public List<Map<String, String>> applyXpathRule(final Document doc) throws CleanerException {

		final List<Map<String, String>> errors = Lists.newArrayList();

		final String id = doc.valueOf("//*[local-name()='objIdentifier']");

		for (Object o : doc.selectNodes(xpath)) {

			final Node node = (Node) o;

			final String oldValue = node.getText().trim();

			final String newValue = calculateNewValue(oldValue);

			if (strict) {

				final Map<String, String> err = verifyValue(newValue);

				if (err != null) {

					errors.add(err);

					if (logCleaningRules.isInfoEnabled()) {

						logCleaningRules.info("[" + newValue + "] is INVALID, " + "RULE: " + toString() + ", " + "RECORD: " + id + ", " + "XPATH: "

								+ this.getXpath());

					}

				}

			}

			if (logCleaningRules.isInfoEnabled() && !newValue.equals(oldValue)) {

				logCleaningRules.info("[" + oldValue + "] => [" + newValue + "], " + toString() + ", " + "RECORD: " + id + ", " + "XPATH: " + this.getXpath());

			}

			node.setText(newValue);

		}

		return errors;

	}

	protected abstract Map<String, String> verifyValue(final String value) throws CleanerException;

	protected abstract String calculateNewValue(final String oldValue) throws CleanerException;

	public String getXpath() {

		return xpath;

	}

	@Required

	public void setXpath(final String xpath) {

		this.xpath = xpath;

	}

	public boolean isStrict() {

		return strict;

	}

	public void setStrict(final boolean strict) {

		this.strict = strict;

	}

}

$inspector/master(it={

<style type="text/css">

  #results {

    width: 100%;

  }

  #results td:first-child {

   width: 2em;

  }

  #results td {

    border: 1px solid #cecece;

  }

</style>

<h2>Browse indices</h2>

<form method="POST">

Cleaner rules:

<select name="rule">

$rules:{<option $if(it.selected)$selected$endif$>$it.value$</option>}$

</select><br /><br />

Dirty Record:<br />

<textarea name="dirtyText" cols="80" rows="10">$dirtyText$</textarea>

<br /><br />

<input type="submit" value="submit"/>

</form>

<br />

Cleaned Record:<br />

<textarea readonly="readonly" cols="80" rows="10">$cleanedText$</textarea>

})$

<?xml version="1.0" encoding="UTF-8"?>

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">

    <xs:annotation>

        <xs:documentation>Standard part of any Resource Profile</xs:documentation>

    </xs:annotation>

    <xs:element name="RESOURCE_PROFILE">

        <xs:complexType>

            <xs:sequence>

                <xs:element type="HEADERType" name="HEADER"/>

                <xs:element type="BODYType" name="BODY"/>

            </xs:sequence>

        </xs:complexType>

    </xs:element>

    <xs:complexType name="HEADERType">

        <xs:all>

            <xs:element name="RESOURCE_IDENTIFIER" type="RESOURCE_IDENTIFIERType"/>

            <xs:element type="RESOURCE_TYPEType" name="RESOURCE_TYPE"/>

            <xs:element type="RESOURCE_KINDType" name="RESOURCE_KIND"/>

            <xs:element type="RESOURCE_URIType" name="RESOURCE_URI"/>

            <xs:element name="DATE_OF_CREATION" type="DATEType"/>

        </xs:all>

    </xs:complexType>

    <xs:complexType name="BODYType">

        <xs:sequence>

            <xs:element name="CONFIGURATION" type="CONFIGURATIONType"/>

            <xs:element type="STATUSType" name="STATUS"/>

            <xs:element ref="SECURITY_PARAMETERS"/>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="RESOURCE_IDENTIFIERType">

        <xs:attribute use="required" name="value" type="xs:string"/>

    </xs:complexType>

    <xs:complexType name="RESOURCE_TYPEType">

        <xs:attribute name="value" use="required">

            <xs:simpleType>

                <xs:restriction base="xs:string">

                    <xs:enumeration value="CleanerDSResourceType"/>

                </xs:restriction>

            </xs:simpleType>

        </xs:attribute>

    </xs:complexType>

    <xs:complexType name="RESOURCE_KINDType">

        <xs:attribute use="required" name="value">

            <xs:simpleType>

                <xs:restriction base="xs:string">

                    <xs:enumeration value="CleanerDSResources"/>

                </xs:restriction>

            </xs:simpleType>

        </xs:attribute>

    </xs:complexType>

    <xs:complexType name="RESOURCE_URIType">

        <xs:attribute name="value" type="xs:string" use="required"/>

    </xs:complexType>

    <xs:complexType name="DATEType">

        <xs:attribute use="required" name="value" type="xs:dateTime"/>

    </xs:complexType>

    <xs:annotation>

        <xs:documentation>Customisable part of any Resource Profile</xs:documentation>

    </xs:annotation>

    <xs:complexType name="CONFIGURATIONType">

        <xs:sequence>

            <xs:element type="xs:string" name="CLEANER_NAME"/>

   			<xs:element type="xs:string" name="CLEANER_DESCRIPTION"/>

            <xs:element type="CLEANERRULESType" name="CLEANER_RULES"/>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="CLEANERRULESType">

        <xs:sequence>

            <xs:element name="RULE" maxOccurs="unbounded" type="RULEType"/>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="RULEType">

        <xs:attribute type="xs:string" name="xpath" use="required"/>

        <xs:attribute type="xs:string" name="vocabularies" use="optional"/>

        <xs:attribute type="xs:string" name="groovy" use="optional" />

        <xs:attribute type="xs:boolean" name="strict" use="optional" />

    </xs:complexType>

    <xs:complexType name="STATUSType">

        <xs:all>

            <xs:element type="DATEType" name="LAST_UPDATE"/>

        </xs:all>

    </xs:complexType>

    <xs:element type="xs:string" name="SECURITY_PARAMETERS" />

</xs:schema>

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

	<bean id="cleanerInspectorGroup"

		class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"

		p:name="cleaner">

		<property name="descriptors">

			<list>

				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"

					p:name="cleaner" p:relativeUrl="cleaner.do"

					p:hiddenAsDefault="true"/>

			</list>

		</property>

	</bean>

</beans>

service.cleaner.mapped.resultset.factory=mappedResultSetFactory

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- beans -->

	<bean id="cleanerService" 

		class="eu.dnetlib.data.utility.cleaner.CleanerServiceImpl"

		init-method="start" destroy-method="stop" p:cleaningRuleFactory-ref="cleaningRuleFactory"

		p:mappedResultSetFactory-ref="${service.cleaner.mapped.resultset.factory}" />

	<bean id="cleaningRuleFactory"

		class="eu.dnetlib.data.utility.cleaner.CleaningRuleFactory" 

		p:lookupLocator-ref="lookupLocator">

	</bean>

	<!-- endpoints -->

	<jaxws:endpoint id="cleanerServiceEndpoint" implementor="#cleanerService"

		implementorClass="eu.dnetlib.data.utility.cleaner.rmi.CleanerService"

		address="/cleaner" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="cleanerServiceRegistrationManager" t:service="cleanerService"

		t:endpoint="cleanerServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

</beans>

<?xml version="1.0" encoding="UTF-8"?>

<beansProjectDescription>

	<version>1</version>

	<pluginVersion><![CDATA[2.2.8.200911091054-RELEASE]]></pluginVersion>

	<configSuffixes>

		<configSuffix><![CDATA[xml]]></configSuffix>

	</configSuffixes>

	<enableImports><![CDATA[false]]></enableImports>

	<configs>

		<config>src/main/eu/dnetlib/data/utility/cleaner/applicationContext-cleaner.xml</config>

	</configs>

	<configSets>

	</configSets>

</beansProjectDescription>

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>0.0.1-alpha</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>cnr-data-utility-cleaner-service</artifactId>

	<packaging>jar</packaging>

	<version>1.0.0</version>

	<dependencies>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.6</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-test-utils</artifactId>

			<version>0.0.2</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-rmi-api</artifactId>

			<version>0.0.9</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>0.1.1</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>0.0.10</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>org.codehaus.groovy</groupId>

			<artifactId>groovy-all</artifactId>

			<version>2.1.6</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-inspector</artifactId>

			<version>[0.0.0,)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-data-utility-cleaner-rmi</artifactId>

			<version>1.0.0</version>

		</dependency>		

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-data-utility-cleaner-service/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "cnr-data-utility-cleaner-service"}

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertNull;

import static org.mockito.Matchers.anyString;

import static org.mockito.Mockito.times;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import java.io.StringReader;

import java.util.List;

import org.dom4j.Document;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.runners.MockitoJUnit44Runner;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

@RunWith(MockitoJUnit44Runner.class)

public class VocabularyRuleTest {

	private static final String VOCABULARY_NAME_1 = "TEST VOCABULARY 1";

	private static final String VOCABULARY_NAME_2 = "TEST VOCABULARY 2";

	private static final List<String> VOCABULARY = Lists.newArrayList("XXXX|-:-|AAAA", "YYYY|-:-|AAAA", "ZZZZ|-:-|AAAA");

	/**

	 * Class Under Test

	 */

	private VocabularyRule rule;

	@Mock

	private ISLookUpService lookup;

	@Before

	public void setUp() throws Exception {

		when(lookup.quickSearchProfile(anyString())).thenReturn(VOCABULARY);

		rule = new VocabularyRule(Sets.newHashSet(VOCABULARY_NAME_1, VOCABULARY_NAME_2), lookup);

	}

	@Test

	public void testSetup() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		execute("<a><b>XXXX</b></a>");

		verify(lookup, times(2)).quickSearchProfile(anyString());

		assertEquals(VOCABULARY.size(), rule.getVocabularyTerms().size());

	}

	@Test

	public void testApplyXpathRule() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_2() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_3() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_attr() throws Exception {

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b value='XXXX' /></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_with_spaces() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>  XXXX  </b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("  XXXX  "));

	}

	@Test

	public void testApplyXpathRule_case() throws Exception {

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute("<a><b>Xxxx</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(rule.verifyValue("AAAA"));

		assertNotNull(rule.verifyValue("Xxxx"));

	}

	private Document execute(final String xml) throws Exception {

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(xml));

		System.out.println("BEFORE:\n" + doc.asXML() + "\n");

		rule.applyXpathRule(doc);

		System.out.println("AFTER:\n" + doc.asXML() + "\n");

		System.out.println("-----------------------------\n");

		return doc;

	}

}

package eu.dnetlib.data.utility.cleaner;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import java.util.HashMap;

import java.util.Map;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnit44Runner;

import com.google.common.collect.Lists;

import eu.dnetlib.data.utility.cleaner.rmi.CleanerException;

@RunWith(MockitoJUnit44Runner.class)

public class XMLCleaningRuleTest {

	private static final String INPUT_VALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>HELLO</a>" + "	</metadata>" + "</record>";

	private static final String INPUT_INVALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>GOOD BYE</a>" + "	</metadata>" + "</record>";

	/**

	 * Class under test.

	 */

	private CleaningRule xmlRule;

	private XPATHCleaningRule mockXpathRule = new XPATHCleaningRule() {

		@Override

		protected Map<String, String> verifyValue(final String value) throws CleanerException {

			if (value.equals("CIAO")) { return null; }

			Map<String, String> err = new HashMap<String, String>();

			err.put("term", value);

			return err;

		}

		@Override

		protected String calculateNewValue(final String oldValue) throws CleanerException {

			if (oldValue.equals("HELLO")) { return "CIAO"; }

			return oldValue;

		}

	};;

	@Before

	public void setUp() throws Exception {

		xmlRule = new CleaningRule();