/ - Diff - D-Net - D-Net project tracking tool

     package eu.dnetlib.data.mapreduce.util;
     import java.util.List;
     import java.util.Map.Entry;
     import com.google.protobuf.Descriptors.EnumValueDescriptor;
     import com.google.protobuf.Descriptors.FieldDescriptor;
     import com.google.protobuf.GeneratedMessage;
     import com.google.protobuf.InvalidProtocolBufferException;
     import com.google.protobuf.Message;
     import eu.dnetlib.data.proto.KindProtos.Kind;
     import eu.dnetlib.data.proto.OafProtos.Oaf;
     import eu.dnetlib.data.proto.OafProtos.OafEntity;
     import eu.dnetlib.data.proto.OafProtos.OafRel;
     import eu.dnetlib.data.proto.RelTypeProtos.RelType;
     /**
      * Helper class, to be used as accessor helper over the Oaf structure.
+     *
      * @author claudio
+     *
      */
     public class OafDecoder {
     	/**
     	 * Oaf object
     	 */
     	private Oaf oaf;
     	/**
     	 * Cached sub decoder
     	 */
     	private OafEntityDecoder entityDecoder = null;
     	/**
     	 * Cached sub decoder
     	 */
     	private OafRelDecoder relDecoder = null;
     	public static OafDecoder decode(final Oaf oaf) {
     		return new OafDecoder(oaf);
+    	}
     	public static OafDecoder decode(final byte[] oaf) {
     		return new OafDecoder(oaf);
+    	}
     	protected OafDecoder(final byte[] value) {
     		try {
     			this.oaf = Oaf.parseFrom(value);
     		} catch (InvalidProtocolBufferException e) {
     			throw new RuntimeException("unable to deserialize proto: " + new String(value));
+    		}
+    	}
     	private OafDecoder(final Oaf oaf) {
     		this.oaf = oaf;
+    	}
     	public Kind getKind() {
     		return oaf.getKind();
+    	}
     	public Oaf getOaf() {
     		return oaf;
+    	}
     	// / Entity
     	public GeneratedMessage getMetadata() {
     		return decodeEntity().getMetadata();
+    	}
     	public GeneratedMessage getOafEntity() {
     		return decodeEntity().getEntity();
+    	}
     	public String getEntityId() {
     		return decodeEntity().getId();
+    	}
     	public OafEntity getEntity() {
     		return oaf.getEntity();
+    	}
     	public OafEntityDecoder decodeEntity() {
     		if (entityDecoder == null) {
     			entityDecoder = OafEntityDecoder.decode(getEntity());
+    		}
     		return entityDecoder;
+    	}
     	// / Rel
     	public OafRel getOafRel() {
     		return oaf.getRel();
+    	}
     	public GeneratedMessage getRel() {
     		return decodeRel().getRel();
+    	}
     	public RelType relType() {
     		return decodeRel().getRelType();
+    	}
     	public String relTypeName() {
     		return relType().toString();
+    	}
     	public String relSourceId() {
     		return decodeRel().getRelSourceId();
+    	}
     	public String relTargetId() {
     		return decodeRel().getRelTargetId();
+    	}
     	public String getCFQ() {
     		switch (getKind()) {
     		case entity:
     			return getEntity().getType().toString();
     		case relation:
     			return decodeRel().getCFQ();
     		default:
     			throw new IllegalArgumentException("Invalid Kind: " + getKind());
+    		}
+    	}
     	public RelDescriptor getRelDescriptor() {
     		if (!getKind().equals(Kind.relation)) return null;
     		return decodeRel().getRelDescriptor();
+    	}
     	private OafRelDecoder decodeRel() {
     		if (relDecoder == null) {
     			relDecoder = OafRelDecoder.decode(getOafRel());
+    		}
     		return relDecoder;
+    	}
     	public byte[] toByteArray() {
     		return oaf.toByteArray();
+    	}
     	public String asXml() {
     		StringBuilder sb = new StringBuilder("<oaf>");
     		for (Entry<FieldDescriptor, Object> e : oaf.getAllFields().entrySet()) {
     			asXml(sb, e.getKey(), e.getValue());
+    		}
     		sb.append("</oaf>");
     		return sb.toString();
+    	}
     	@SuppressWarnings("unchecked")
     	private void asXml(final StringBuilder sb, final FieldDescriptor fd, final Object value) {
     		if (fd.isRepeated() && (value instanceof List<?>)) {
     			for (Object o : ((List<Object>) value)) {
     				asXml(sb, fd, o);
+    			}
     		} else if (fd.getType().equals(FieldDescriptor.Type.MESSAGE)) {
     			sb.append("<" + fd.getName() + ">");
     			for (Entry<FieldDescriptor, Object> e : ((Message) value).getAllFields().entrySet()) {
     				asXml(sb, e.getKey(), e.getValue());
+    			}
     			sb.append("</" + fd.getName() + ">");
     		} else if (fd.getType().equals(FieldDescriptor.Type.ENUM)) {
     			sb.append("<" + fd.getName() + ">");
     			sb.append(((EnumValueDescriptor) value).getName());
     			sb.append("</" + fd.getName() + ">");
     		} else {
     			sb.append("<" + fd.getName() + ">");
     			sb.append(escapeXml(value.toString()));
     			sb.append("</" + fd.getName() + ">");
+    		}
+    	}
     	private static String escapeXml(final String value) {
     		return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
+    	}
+    }

     package eu.dnetlib.data.index;
     import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;
     import eu.dnetlib.miscutils.datetime.HumanTime;
     import eu.dnetlib.miscutils.functional.UnaryFunction;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.apache.solr.client.solrj.SolrQuery;
     import org.apache.solr.client.solrj.SolrServerException;
     import org.apache.solr.client.solrj.impl.CloudSolrClient;
     import org.apache.solr.client.solrj.response.QueryResponse;
     import org.apache.solr.client.solrj.response.UpdateResponse;
     import org.apache.solr.common.SolrInputDocument;
     import java.io.Closeable;
     import java.io.IOException;
     import java.text.SimpleDateFormat;
     import java.util.Date;
     import java.util.List;
     /**
      * Created by michele on 11/11/15.
      */
     public class CloudIndexClient implements Closeable {
     	private static final Log log = LogFactory.getLog(CloudIndexClient.class);
     	private static final String INDEX_RECORD_RESULT_FIELD = "dnetResult";
     	private final CloudSolrClient solrClient;
     	protected CloudIndexClient(final CloudSolrClient solrServer) {
     		this.solrClient = solrServer;
+    	}
     	public int feed(final String record, final String indexDsId, final UnaryFunction<String, String> toIndexRecord) throws CloudIndexClientException {
     		return feed(record, indexDsId, toIndexRecord, true);
+    	}
     	public int feed(final String record, final String indexDsId, final UnaryFunction<String, String> toIndexRecord, final boolean commit)
     			throws CloudIndexClientException {
     		try {
     			final SolrInputDocument doc = prepareSolrDocument(record, indexDsId, toIndexRecord);
     			if ((doc == null) || doc.isEmpty()) throw new CloudIndexClientException("Invalid solr document");
     			return feed(doc, commit);
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error feeding document", e);
+    		}
+    	}
     	public int feed(final SolrInputDocument document) throws CloudIndexClientException {
     		return feed(document, true);
+    	}
     	public int feed(final SolrInputDocument document, final boolean commit) throws CloudIndexClientException {
     		try {
     			final UpdateResponse res = solrClient.add(document);
     			log.debug("feed time for single records, elapsed time: " + HumanTime.exactly(res.getElapsedTime()));
     			if (res.getStatus() != 0) { throw new CloudIndexClientException("bad status: " + res.getStatus()); }
     			if (commit) {
     				solrClient.commit();
+    			}
     			return res.getStatus();
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error feeding document", e);
+    		}
+    	}
     	public void feed(final List<SolrInputDocument> docs, final AfterFeedingCallback callback) throws CloudIndexClientException {
     		feed(docs, callback, true);
+    	}
     	public void feed(final List<SolrInputDocument> docs, final AfterFeedingCallback callback, final boolean commit) throws CloudIndexClientException {
     		try {
     			if (docs.isEmpty()) {
     				log.debug("Empty list of documents. Calling callback, if needed.");
     				if (callback != null) {
     					callback.doAfterFeeding(null);
+    				}
     				return;
+    			}
     			final UpdateResponse res = solrClient.add(docs);
     			log.debug("feed time for " + docs.size() + " records, elapsed tipe: : " + HumanTime.exactly(res.getElapsedTime()));
     			if (commit) {
     				solrClient.commit();
+    			}
     			if (callback != null) {
     				callback.doAfterFeeding(res);
+    			}
     			if (res.getStatus() != 0) throw new CloudIndexClientException("bad status: " + res.getStatus());
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error feeding documents", e);
+    		}
+    	}
     	public SolrInputDocument prepareSolrDocument(final String record, final String indexDsId, final UnaryFunction<String, String> toIndexRecord)
     			throws CloudIndexClientException {
     		try {
     			final StreamingInputDocumentFactory documentFactory = new StreamingInputDocumentFactory();
     			final String version = (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date());
     			final String indexRecord = toIndexRecord.evaluate(record);
     			if (log.isDebugEnabled()) {
     				log.debug("***************************************\nSubmitting index record:\n" + indexRecord + "\n***************************************\n");
+    			}
     			return documentFactory.parseDocument(version, indexRecord, indexDsId, INDEX_RECORD_RESULT_FIELD);
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error creating solr document", e);
+    		}
+    	}
     	public boolean isRecordIndexed(final String id) throws CloudIndexClientException {
     		final QueryResponse res = query("objidentifier:\"" + id + "\"", null);
     		return res.getResults().size() > 0;
+    	}
     	public int remove(final String id) throws CloudIndexClientException {
     		return remove(id, true);
+    	}
     	public int remove(final String id, final boolean commit) throws CloudIndexClientException {
     		try {
     			final UpdateResponse res = solrClient.deleteByQuery("objidentifier:\"" + id + "\"");
     			if (commit) {
     				solrClient.commit();
+    			}
     			return res.getResponse().size();
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error removing documents", e);
+    		}
+    	}
     	public int count(final String query) throws CloudIndexClientException {
     		final QueryResponse res = query(query, 0);
     		return res.getResults().size();
+    	}
     	public QueryResponse query(final String query, Integer rows) throws CloudIndexClientException {
     		try {
     			final SolrQuery solrQuery = new SolrQuery();
     			solrQuery.setQuery(query);
     			if(rows != null && rows >= 0) {
     				solrQuery.setRows(rows);
+    			}
     			return solrClient.query(solrQuery);
     		} catch (final Throwable e) {
     			throw new CloudIndexClientException("Error searching documents", e);
+    		}
+    	}
     	public void close() throws IOException {
     		if (solrClient != null) {
     			solrClient.close();
+    		}
+    	}
     	public void commit() throws CloudIndexClientException {
     		if(solrClient != null) {
     			try {
     				solrClient.commit();
     			} catch (SolrServerException | IOException e) {
     				throw new CloudIndexClientException(e.getMessage());
+    			}
+    		}
+    	}
     	public interface AfterFeedingCallback {
     		void doAfterFeeding(final UpdateResponse response);
+    	}
+    }

     package eu.dnetlib.data.bulktag;
     import com.google.common.collect.Lists;
     import com.google.common.collect.Maps;
     import com.google.gson.Gson;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import java.util.ArrayList;
     import java.util.HashMap;
     import java.util.List;
     import java.util.Map;
     import java.util.stream.Collectors;
     /**
      * Created by miriam on 02/08/2018.
      */
     public class CommunityConfiguration {
         private static final Log log = LogFactory.getLog(CommunityConfiguration.class);
         enum MapModes{
             SUBJECT_MAP,
             DATASOURCE_MAP,
             ZENODO_COMMUNITY_MAP
+        }
         private Map<String,Community> communities;
         //map subject -> communityid
         private transient Map<String,List<Pair<String,SelectionCriteria>>> subjectMap = new HashMap<>();
         //map datasourceid -> communityid
         private transient Map<String,List<Pair<String,SelectionCriteria>>> datasourceMap = new HashMap<>();
         //map zenodocommunityid -> communityid
         private transient Map<String,List<Pair<String,SelectionCriteria>>> zenodocommunityMap = new HashMap<>();
         public CommunityConfiguration(final Map<String, Community> communities) {
             this.communities = communities;
             init();
+        }
         public void init() {
             if (subjectMap == null) {
                 subjectMap = Maps.newHashMap();
+            }
             if (datasourceMap == null) {
                 datasourceMap = Maps.newHashMap();
+            }
             if (zenodocommunityMap == null) {
                 zenodocommunityMap = Maps.newHashMap();
+            }
             for(Community c : getCommunities().values()) {
                 //get subjects
                 final String id = c.getId();
                 for(String sbj : c.getSubjects()){
                     Pair<String,SelectionCriteria> p = new Pair<>(id,new SelectionCriteria(null));
                     add(sbj.toLowerCase().trim() , p, subjectMap);
+                }
                 //get datasources
                 for(Datasource d: c.getDatasources()){
                     add(d.getOpenaireId(),new Pair<>(id,d.getSelCriteria()),datasourceMap);
+                }
                 //get zenodo communities
                 for(ZenodoCommunity zc : c.getZenodoCommunities()){
                     add(zc.getZenodoCommunityId(),new Pair<>(id,zc.getSelCriteria()),zenodocommunityMap);
+                }
+            }
+        }
         private void add(String key,Pair<String,SelectionCriteria> value, Map<String,List<Pair<String,SelectionCriteria>>> map){
             List<Pair<String,SelectionCriteria>> values = map.get(key);
             if (values == null){
                 values = new ArrayList<>();
                 map.put(key,values);
+            }
             values.add(value);
+        }
         public List<Pair<String,SelectionCriteria>> getCommunityForSubject(String sbj){
             return subjectMap.get(sbj);
+        }
         public List<Pair<String,SelectionCriteria>> getCommunityForDatasource(String dts){
             return datasourceMap.get(dts);
+        }
         public List<Pair<String,SelectionCriteria>> getCommunityForZenodoCommunity(String zc){
             return zenodocommunityMap.get(zc);
+        }
         public List<String> getCommunityForSubjectValue(String value) {
             return getContextIds(subjectMap.get(value));
+        }
         public List<String> getCommunityForDatasourceValue(String value) {
             return getContextIds(datasourceMap.get(value.toLowerCase()));
+        }
         public List<String> getCommunityForZenodoCommunityValue(String value){
             return getContextIds(zenodocommunityMap.get(value.toLowerCase()));
+        }
         private List<String> getContextIds(List<Pair<String, SelectionCriteria>> list) {
             if (list != null) {
                 return list.stream().map(p -> p.getFst()).collect(Collectors.toList());
+            }
             return Lists.newArrayList();
+        }
         /*
         public SelectionCriteria getSelCriteria(String value, String community, MapModes map_mode){
             Map<String,List<Pair<String,SelectionCriteria>>> map = null;
             if(map_mode == MapModes.DATASOURCE_MAP)
                 map = datasourceMap;
             else
             if(map_mode == MapModes.ZENODO_COMMUNITY_MAP)
                 map = zenodocommunityMap;
             else
                 new Throwable("Impossible to have Selection Criteria over subjects");
             List<Pair<String, SelectionCriteria>> lst = map.get(value);
             List<SelectionCriteria> selectionList = lst.stream().map(p -> {
                 if (p.fst == community)
                     return p.snd;
                 return null;
             }).collect(Collectors.toList());//for each community there will be only one Selection Criteria per datasource or zenodo community
             if(selectionList != null)
                 if (selectionList.size()>0)
                     return selectionList.get(0);
             return null;
+        }
         */
         public Map<String, Community> getCommunities() {
             return communities;
+        }
         public void setCommunities(Map<String, Community> communities) {
             this.communities = communities;
+        }
         public String toJson() {
             final Gson g = new Gson();
             return g.toJson(this);
+        }
         public int size() {
             return communities.keySet().size();
+        }
         public Community getCommunityById(String id){
             return communities.get(id);
+        }
         public List<Community> getCommunityList() {
             return Lists.newLinkedList(communities.values());
+        }
+    }

     <?xml version="1.0" encoding="UTF-8"?>
     <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     	<parent>
     		<groupId>eu.dnetlib</groupId>
     		<artifactId>dnet45-parent</artifactId>
     		<version>1.0.0</version>
     		<relativePath />
     	</parent>
     	<modelVersion>4.0.0</modelVersion>
     	<groupId>eu.dnetlib</groupId>
     	<artifactId>dnet-openaireplus-mapping-utils</artifactId>
     	<packaging>jar</packaging>
     	<version>6.3.24</version>
     	<scm>
     		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.24</developerConnection>
     	</scm>
     	<properties>
     		<skipITs>true</skipITs>
     	</properties>
     	<build>
     		<plugins>
     			<plugin>
     				<groupId>org.apache.maven.plugins</groupId>
     				<artifactId>maven-failsafe-plugin</artifactId>
     				<version>2.19.1</version>
     				<executions>
     					<execution>
     						<id>integration-test</id>
     						<goals>
     							<goal>integration-test</goal>
     						</goals>
     					</execution>
     					<execution>
     						<id>verify</id>
     						<goals>
     							<goal>verify</goal>
     						</goals>
     					</execution>
     				</executions>
     				<configuration>
     					<skipITs>${skipITs}</skipITs>
     				</configuration>
     			</plugin>
     		</plugins>
     	</build>
     	<dependencies>
     		<dependency>
     			<groupId>com.google.guava</groupId>
     			<artifactId>guava</artifactId>
     			<version>${google.guava.version}</version>
     		</dependency>
     		<dependency>
     			<groupId>junit</groupId>
     			<artifactId>junit</artifactId>
     			<version>${junit.version}</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>com.ximpleware</groupId>
     			<artifactId>vtd-xml</artifactId>
     			<version>[2.12, 3.0.0)</version>
     		</dependency>
     		<dependency>
     			<groupId>commons-codec</groupId>
     			<artifactId>commons-codec</artifactId>
     			<version>${commons.codec.version}</version>
     		</dependency>
     		<dependency>
     			<groupId>dom4j</groupId>
     			<artifactId>dom4j</artifactId>
     			<version>${dom4j.version}</version>
     			<exclusions>
     				<exclusion>
     					<artifactId>xml-apis</artifactId>
     					<groupId>xml-apis</groupId>
     				</exclusion>
     			</exclusions>
     		</dependency>
     		<dependency>
     			<groupId>net.sf.supercsv</groupId>
     			<artifactId>super-csv</artifactId>
     			<version>2.4.0</version>
     		</dependency>
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>dnet-openaire-data-protos</artifactId>
     			<version>[3.9.3]</version>
     		</dependency>
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>dnet-pace-core</artifactId>
     			<version>[3.0.0,4.0.0)</version>
     		</dependency>
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>cnr-misc-utils</artifactId>
     			<version>[1.0.0,2.0.0)</version>
     		</dependency>
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>dnet-hadoop-commons</artifactId>
     			<version>[2.0.0,3.0.0)</version>
     		</dependency>
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>dnet-index-solr-common</artifactId>
     			<version>[2.3.4,3.0.0)</version>
     		</dependency>
     		<dependency>
     			<groupId>com.googlecode.protobuf-java-format</groupId>
     			<artifactId>protobuf-java-format</artifactId>
     			<version>1.2</version>
     		</dependency>
     		<dependency>
     			<groupId>org.apache.commons</groupId>
     			<artifactId>commons-lang3</artifactId>
     			<version>3.5</version>
     		</dependency>
     		<!-- test deps -->
     		<dependency>
     			<groupId>eu.dnetlib</groupId>
     			<artifactId>dnet-openaireplus-profiles</artifactId>
     			<version>[1.0.0,2.0.0)</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>org.mongodb</groupId>
     			<artifactId>mongo-java-driver</artifactId>
     			<version>${mongodb.driver.version}</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>org.springframework</groupId>
     			<artifactId>spring-context</artifactId>
     			<version>${spring.version}</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>org.springframework</groupId>
     			<artifactId>spring-core</artifactId>
     			<version>${spring.version}</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>org.springframework</groupId>
     			<artifactId>spring-test</artifactId>
     			<version>${spring.version}</version>
     			<scope>test</scope>
     		</dependency>
     	</dependencies>
     </project>

     package eu.dnetlib.data.transform.xml.vtd;
     import java.io.IOException;
     import java.util.function.Function;
     import eu.dnetlib.data.proto.OafProtos.Oaf;
     import eu.dnetlib.data.transform.xml2.DatasetToProto;
     import eu.dnetlib.data.transform.xml2.PublicationToProto;
     import org.apache.commons.io.IOUtils;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.junit.Test;
     import static org.junit.Assert.assertNotNull;
     public class VtdParserToProtoTest {
     	private static final Log log = LogFactory.getLog(VtdParserToProtoTest.class);
     	@Test
     	public void testParsePublication() throws IOException {
     		doTest("/eu/dnetlib/data/transform/publication.xml", new PublicationToProto());
+    	}
     	@Test
     	public void testParseDataset() throws IOException {
     		doTest("/eu/dnetlib/data/transform/dataset.xml", new DatasetToProto());
+    	}
     	@Test
     	public void testParseDataset2() throws IOException {
     		doTest("/eu/dnetlib/data/transform/dataset2.xml", new DatasetToProto());
+    	}
     	private void doTest(final String filePath, Function<String, Oaf> f) throws IOException {
     		final String xml = IOUtils.toString(getClass().getResourceAsStream(filePath));
     		assertNotNull(xml);
     		final Oaf oaf = f.apply(xml);
     		assertNotNull(oaf);
     		log.info(oaf.toString());
+    	}
+    }

     package eu.dnetlib.data.transform.xml.vtd;
     import java.io.IOException;
     import java.io.InputStream;
     import java.util.Properties;
     import com.mongodb.MongoClient;
     import com.mongodb.client.MongoDatabase;
     import org.springframework.context.annotation.Bean;
     import org.springframework.context.annotation.Configuration;
     import org.springframework.core.io.ClassPathResource;
     @Configuration
     public class ConfigurationTestConfig {
     	@Bean
     	public MongoDatabase db() throws IOException {
     		final Properties p = new Properties();
     		final ClassPathResource cp = new ClassPathResource("/eu/dnetlib/data/transform/xml/vtd/test.properties");
     		try (final InputStream stream = cp.getInputStream()) {
     			p.load(stream);
+    		}
     		final MongoClient mongo = new MongoClient(
     				p.getProperty("mongodb.host"),
     				Integer.valueOf(p.getProperty("mongodb.port")));
     		return mongo.getDatabase(p.getProperty("mongodb.dbname"));
+    	}
+    }

     package eu.dnetlib.data.transform.xml.vtd;
     import com.mongodb.client.MongoCollection;
     import com.mongodb.client.MongoDatabase;
     import eu.dnetlib.data.transform.xml2.DatasetToProto;
     import org.apache.commons.lang3.time.StopWatch;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
     import org.bson.Document;
     import org.junit.Test;
     import org.junit.runner.RunWith;
     import org.springframework.beans.factory.annotation.Autowired;
     import org.springframework.test.context.ContextConfiguration;
     import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
     import java.io.IOException;
     import java.util.Objects;
     import java.util.concurrent.atomic.AtomicInteger;
     import java.util.stream.StreamSupport;
     import static org.junit.Assert.assertNotNull;
     import static org.junit.Assert.assertTrue;
     @RunWith(SpringJUnit4ClassRunner.class)
     @ContextConfiguration(classes = { ConfigurationTestConfig.class })
     public class VtdParserToProtoIT {
     	private static final Log log = LogFactory.getLog(VtdParserToProtoIT.class);
     	public static final String COLLECTION_NAME = "datacite";
     	private static final int BATCH_SIZE = 10000;
     	public static final int LOG_FREQ = 5000;
     	@Autowired
     	private MongoDatabase db;
     	@Test
     	public void testParseDatacite() throws IOException {
     		final MongoCollection<Document> collection = db.getCollection(COLLECTION_NAME);
     		final long collectionSize = collection.count();
     		log.info(String.format("found %s records in collection '%s'", collectionSize, COLLECTION_NAME));
     		final AtomicInteger read = new AtomicInteger(0);
     		final DescriptiveStatistics stats = new DescriptiveStatistics();
     		final StopWatch timer = new StopWatch();
     		final DatasetToProto mapper = new DatasetToProto();
     		StreamSupport.stream(collection.find().batchSize(BATCH_SIZE).spliterator(), false)
     				.peek(d -> {
     					if (read.addAndGet(1) % LOG_FREQ == 0) {
     						log.info(String.format("records read so far %s", read.get()));
     						log.info(String.format("stats so far %s", stats.toString()));
+    					}
     				})
     				.map(d -> (String) d.get("body"))
     				.filter(Objects::nonNull)
     				.peek(s -> timer.start())
     				.map(mapper)
     				.forEach(oaf -> {
     					assertNotNull(oaf);
     					assertTrue(oaf.hasEntity());
     					timer.stop();
     					stats.addValue(timer.getTime());
     					timer.reset();
     				});
     		log.info(String.format("processed %s/%s records", read.get(), collectionSize));
+    	}
+    }

     package eu.dnetlib.data.transform.xml.vtd;
     import java.io.InputStream;
     import java.util.List;
     import com.ximpleware.AutoPilot;
     import com.ximpleware.VTDGen;
     import com.ximpleware.VTDNav;
     import eu.dnetlib.data.transform.xml2.Node;
     import eu.dnetlib.data.transform.xml2.VtdUtilityParser;
     import org.apache.commons.io.IOUtils;
     import org.junit.Assert;
     import org.junit.Test;
     public class VtdUtilityParserTest {
     	@Test
     	public void testUtils1() {
     		String xpath = VtdUtilityParser.xpath("a", "b", "c");
     		Assert.assertTrue("/*[local-name()='a']/*[local-name()='b']/*[local-name()='c']".equals(xpath));
+    	}
         @Test
         public void testPartser() throws Exception {
             final InputStream resource = this.getClass().getResourceAsStream("/eu/dnetlib/data/transform/publication.xml");
             final String record =IOUtils.toString(resource);
             final VTDGen vg = VtdUtilityParser.parseXml(record);
             final VTDNav vn = vg.getNav();
             final AutoPilot ap = new AutoPilot(vn);
             List<Node> nodes = VtdUtilityParser.getNodes(ap, vn, "//*[local-name()='referenceaa']");
             nodes.forEach(n -> Assert.assertTrue(n.getAttributes().keySet().size()>0));
             System.out.println(VtdUtilityParser.countNodes(ap, vn, "count(//*[local-name()='CobjIdentifier'])"));
+        }
+    }

     package eu.dnetlib.data.transform.xml;
     import java.util.List;
     import eu.dnetlib.data.transform.xml.OpenTrialsXsltFunctions.JsonProv;
     import org.junit.After;
     import org.junit.Before;
     import org.junit.Test;
     import static org.junit.Assert.assertEquals;
     /**
      * OpenTrialsXsltFunctions Tester.
+     *
      */
     public class OpenTrialsXsltFunctionsTest {
     	private String jsonProv = "[{\"url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";
     	private String jsonProvWithNull = "[{\"url\" : \"\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";
     	private String jidentifiers = "{112683,NCT00920439}";
     	private String jsonRecord = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : true},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";
     	private String jsonRecordNull = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\"},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";
     	private String jsonRecordVoid = "[{\"source_id\" : \"\", \"source_url\" : \"\", \"is_primary\" : \"\"}]";
     	private String jsonRecondPrimary = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : false},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";
     	private String jsonPeopleVoid ="[{\"person_name\" : null, \"person_id\" : null, \"person_role\" : null}]";
     	private String jsonPeopleOne = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"}]";
     	private String jsonPeopleMore = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"},{\"person_name\" : \"Miriam Pippolippo Baglioni, PhD\", \"person_id\" : \"fake\", \"person_role\" : \"principal_investigator\"}]";
     	private String jsonOrganizationVoid = "[{\"organization_name\" : null, \"organization_id\" : null, \"organization_role\" : null}]";
     	private String jsonOrganizationOne = "[{\"organization_name\" : \"SÃ¶dertÃ¤lje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"}]";
     	private String jsonOrganizationMore = "[{\"organization_name\" : \"SÃ¶dertÃ¤lje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"},{\"organization_name\" : \"Miriam Baglioni AB\", \"organization_id\" : \"fake\", \"organization_role\" : \"primary_sponsor\"}]";
     	private String jsonLocationVoid = "[{\"location_name\" : null}]";
     	private String jsonLocationOne = "[{\"location_name\" : \"China\"}]";
     	private String jsonLocationMore = "[{\"location_name\" : \"China\"},{\"location_name\" : \"North Korea\"}]";
     	@Before
     	public void before() throws Exception {
+    	}
     	@After
     	public void after() throws Exception {
+    	}
     	/**
     	 * Method: getProvs(String jsonProvList)
     	 */
     	@Test
     	public void testGetProvs() throws Exception {
     		List<JsonProv> list = OpenTrialsXsltFunctions.getProvs(jsonProv);
     		assertEquals(2, list.size());
+    	}
     	/**
     	 * Method: getMainIdentifierURL(String jsonProvList)
     	 */
     	@Test
     	public void testGetMainIdentifierURL() throws Exception {
     		String url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProv);
     		assertEquals( "http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508", url );
     		url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProvWithNull);
     		assertEquals("https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true", url);
+    	}
     	@Test
     	public void testGetPrimaryRecordUrl(){
     		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecord);
     		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);
+    	}
     	@Test
     	public void testGetPrimaryRecordID(){
     		String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecord);
     		assertEquals("nct", id);
+    	}
     	@Test
     	public void testGetPrimaryRecordUrlNull(){
     		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordNull);
     		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);
+    	}
     	@Test
     	public void testGetPrimaryRecordUrlVoid(){
     		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordVoid);
     		assertEquals("", url);
+    	}
     	@Test
     	public void testGetPrimaryRecordUrlNoPrimary(){
     		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecondPrimary);
     		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);
+    	}
     	@Test
     	public void testGetPrimaryRecordIDNoPrimary(){
     		String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecondPrimary);
     		assertEquals("nct", id);
+    	}
     	@Test
     	public void testGetPrincipalInvestigatorsVoid(){
     		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleVoid);
     		assertEquals("",url);
+    	}
     	@Test
     	public void testGetPrincipalInvestigatorsOne(){
     		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleOne);
     		assertEquals("Verheul, Henk", url);
+    	}
     	@Test
     	public void testGetPrincipalInvestigatorsMore(){
     		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleMore);
     		assertEquals("Verheul, Henk@@Baglioni, Miriam Pippolippo", url);
+    	}
     	@Test
     	public void testgGetTrialOrganizationsVoid(){
     		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationVoid);
     		assertEquals("",url);
+    	}
     	@Test
     	public void testgGetTrialOrganizationsOne(){
     		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationOne);
     		assertEquals("SÃ¶dertÃ¤lje sjukhus AB@sponsor", url);
+    	}
     	@Test
     	public void testgGetTrialOrganizationsMore(){
     		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationMore);
     		assertEquals("SÃ¶dertÃ¤lje sjukhus AB@sponsor@@Miriam Baglioni AB@sponsor", url);
+    	}
     	@Test
     	public void testgGetTrialLocationsVoid(){
     		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationVoid);
     		assertEquals("",url);
+    	}
     	@Test
     	public void testgGetTrialLocationsOne(){
     		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationOne);
     		assertEquals("China", url);
+    	}
     	@Test
     	public void testgGetTrialLocationsMore(){
     		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationMore);
     		assertEquals("China@@North Korea", url);
+    	}
     	@Test
     	public void testGetNotPrimaryRecordUrlPrimary(){
     		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecondPrimary);
     		assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url);
+    	}
     	@Test
     	public void testGetNotPrimaryRecordUrlVoid(){
     		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecordVoid);
     		assertEquals("", url);
+    	}
     	@Test
     	public void testGetNotPrimaryRecordUrl(){
     		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecord);
     		assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url);
+    	}
+    }

     package eu.dnetlib.data.transform.xml;
     import org.junit.After;
     import org.junit.Before;
     import org.junit.Test;
     import static org.junit.Assert.assertEquals;
     import static org.junit.Assert.assertNotNull;
     /**
+     *
      * Created by miriam on 04/05/2017.
      */
     public class FWFXsltFunctionsTest {
     	private String namesurname ="Gerhard SOMMER";
     	private String noSurname = "Gerhard";
     	private String noName = "SOMMER";
     	private String twoNames = "Gerhard Pippo SOMMER";
     	private String twoSurname = "Gerhard PIPPO SOMMER";
     	private String nonamesurname = "";
     	private String organization ="UniversitÃ¤t Linz - Institut fÃ¼r Computational Perception; UniversitÃ¤t fÃ¼r Musik und darstellende Kunst Graz - Institut 1: Komposition, Musiktheorie, Musikgeschichte und Dirigieren; UniversitÃ¤t Mozarteum Salzburg - Institut fÃ¼r Musikalische Rezeptions- und Interpretationsgeschichte; Anton Bruckner PrivatuniversitÃ¤t - Institut fÃ¼r Theorie und Geschichte der Musik; Eliette und Herbert von Karajan Institut - Eliette und Herbert von Karajan Institut";
     	@Before
     	public void before() throws Exception {
+    	}
     	@After
     	public void after() throws Exception {
+    	}
     	@Test
     	public void testGetNamesNameNoNameSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(nonamesurname,true);
     		assertEquals("",ret );
+    	}
     	@Test
     	public void testGetNamesSurnameNoNameSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(nonamesurname,false);
     		assertEquals("",ret );
+    	}
     	@Test
     	public void testGetNamesNameTwoSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(twoSurname,true);
     		assertEquals("Gerhard",ret );
+    	}
     	@Test
     	public void testGetNamesSurnameTwoSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(twoSurname,false);
     		assertEquals("PIPPO SOMMER",ret );
+    	}
     	@Test
     	public void testGetNamesNameTwoNames() throws Exception {
     		String ret = FWFXsltFunctions.getName(twoNames,true);
     		assertEquals("Gerhard Pippo",ret );
+    	}
     	@Test
     	public void testGetNamesSurnameTwoNames() throws Exception {
     		String ret = FWFXsltFunctions.getName(twoNames,false);
     		assertEquals("SOMMER",ret );
+    	}
     	/**
     	 * Method: getProvs(String jsonProvList)
     	 */
     	@Test
     	public void testGetNamesName() throws Exception {
     		String ret = FWFXsltFunctions.getName(namesurname,true);
     		assertEquals("Gerhard",ret );
+    	}
     	@Test
     	public void testGetNamesSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(namesurname,false);
     		assertEquals("SOMMER",ret );
+    	}
     	@Test
     	public void testGetNamesNameNoSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(noSurname,true);
     		assertEquals("Gerhard",ret );
+    	}
     	@Test
     	public void testGetNamesSurnameNoSurname() throws Exception {
     		String ret = FWFXsltFunctions.getName(noSurname,false);
     		assertEquals("",ret );
+    	}
     	@Test
     	public void testGetNamesNameNoName() throws Exception {
     		String ret = FWFXsltFunctions.getName(noName,true);
     		assertEquals("",ret );
+    	}
     	@Test
     	public void testGetNamesSurnameNoName() throws Exception {
     		String ret = FWFXsltFunctions.getName(noName,false);
     		assertEquals("SOMMER",ret );
+    	}
     	@Test
     	public void TestGetMd5()throws Exception{
     		String md5 = FWFXsltFunctions.getMd5(organization);
     		System.out.println(md5);
     		assertNotNull(md5);
+    	}
+    }

     package eu.dnetlib.data.bulktag;
     import com.google.gson.Gson;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import java.util.ArrayList;
     import java.util.List;
     /**
      * Created by miriam on 01/08/2018.
      */
     public class Community {
         private static final Log log = LogFactory.getLog(Community.class);
         private String id;
         private List<String> subjects = new ArrayList<>();
         private List<Datasource> datasources = new ArrayList<>();
         private List<ZenodoCommunity> zenodoCommunities = new ArrayList<>();
         public String toJson() {
             final Gson g = new Gson();
             return g.toJson(this);
+        }
         public boolean isValid() {
             return !getSubjects().isEmpty() || !getDatasources().isEmpty() || !getZenodoCommunities().isEmpty();
+        }
         public String getId() {
             return id;
+        }
         public void setId(String id) {
             this.id = id;
+        }
         public List<String> getSubjects() {
             return subjects;
+        }
         public void setSubjects(List<String> subjects) {
             this.subjects = subjects;
+        }
         public List<Datasource> getDatasources() {
             return datasources;
+        }
         public void setDatasources(List<Datasource> datasources) {
             this.datasources = datasources;
+        }
         public List<ZenodoCommunity> getZenodoCommunities() {
             return zenodoCommunities;
+        }
         public void setZenodoCommunities(List<ZenodoCommunity> zenodoCommunities) {
             this.zenodoCommunities = zenodoCommunities;
+        }
+    }

     package eu.dnetlib.data.transform.xml;
     import java.util.HashMap;
     import java.util.List;
     import com.google.common.collect.Iterables;
     import com.google.common.collect.Lists;
     import com.google.protobuf.Descriptors.Descriptor;
     import eu.dnetlib.data.mapreduce.util.OafRowKeyDecoder;
     import eu.dnetlib.data.proto.FieldTypeProtos.Author;
     import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;
     import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;
     import eu.dnetlib.data.proto.OafProtos.Oaf;
     import eu.dnetlib.data.proto.OafProtos.OafEntity;
     import eu.dnetlib.data.proto.ResultProtos.Result;
     import eu.dnetlib.data.proto.ResultProtos.Result.Context;
     import eu.dnetlib.data.proto.ResultProtos.Result.ExternalReference;
     import eu.dnetlib.data.proto.ResultProtos.Result.Instance;
     import eu.dnetlib.data.proto.TypeProtos.Type;
     import org.apache.commons.lang3.StringUtils;
     import org.w3c.dom.NodeList;
     public class OafToHbaseXsltFunctions extends CommonDNetXsltFunctions {
     	public static String oafResult(
     			final String resultId,
     			final boolean invisible,
     			final String provenance,
     			final String trust,
     			final NodeList about,
     			final String originalId,
     			final String dateOfCollection,
     			final String dateOfTransformation,
     			final NodeList metadata) {
     		ValueMap values = null;
     		try {
     			final String entityId = OafRowKeyDecoder.decode(resultId).getKey();
     			values = ValueMap.parseNodeList(metadata);
     			final Descriptor mDesc = Result.Metadata.getDescriptor();
     			final List<KeyValue> collectedFrom = getKeyValues(values, "collectedfrom", Type.datasource);
     			final List<KeyValue> hostedBy = getKeyValues(values, "hostedby", Type.datasource);
     			final Result.Metadata.Builder metadataBuilder = buildMetadata(values, mDesc);
     			final Result.Builder result = buildResult(metadataBuilder, values, collectedFrom, hostedBy);
     			final OafEntity.Builder entity = buildOafEntity(result, entityId, metadata, collectedFrom, originalId);
     			entity.setDateofcollection(dateOfCollection)
     					.setDateoftransformation(dateOfTransformation).setOaiprovenance(getOAIProvenance(about));
     			final Oaf oaf = getOaf(entity, getDataInfo(invisible, about, provenance, trust, false, false));
     			return base64(oaf.toByteArray());
     		} catch (final Throwable e) {
     			handleException(e, resultId, values);
+    		}
     		return null;
+    	}
     	public static String oafResultUpdate(final String resultId,
     			final String provenance,
     			final String trust,
     			final NodeList nodelist) {
     		ValueMap values = null;
     		try {
     			final String entityId = OafRowKeyDecoder.decode(resultId).getKey();
     			values = ValueMap.parseNodeList(nodelist);
     			final List<KeyValue> hostedBy = getKeyValues(values, "hostedby", Type.datasource);
     			final Descriptor mDesc = Result.Metadata.getDescriptor();
     			final Result.Metadata.Builder metadata = buildMetadata(values, mDesc);
     			final Result.Builder result = buildResult(metadata, values, null, hostedBy);
     			final OafEntity.Builder entity = buildOafEntity(result, entityId, nodelist, null, null);
     			final Oaf oaf = getOaf(entity, null);
     			return base64(oaf.toByteArray());
     		} catch (final Throwable e) {
     			handleException(e, resultId, values);
+    		}
     		return null;
+    	}
     	private static OafEntity.Builder buildOafEntity(
     			final Result.Builder result,
     			final String entityId,
     			final NodeList nodelist,
     			final List<KeyValue> collectedFrom,
     			final String originalId) {
     		final List<StructuredProperty> pids = Lists.newArrayList();
     		pids.addAll(parsePids(nodelist));
     		final OafEntity.Builder entity =
     				getEntity(Type.result, entityId, collectedFrom, StringUtils.isBlank(originalId) ? null : Lists.newArrayList(originalId), null, null, pids)
     						.setResult(result);
     		return entity;
+    	}
     	private static Result.Metadata.Builder buildMetadata(final ValueMap values, final Descriptor mDesc) {
     		final Result.Metadata.Builder metadata = Result.Metadata.newBuilder();
     		if (values.get("creator") != null) {
     			for (final Element e : values.get("creator")) {
     				final Author.Builder author = Author.newBuilder();
     				final String fullname = e.getText();
     				author.setFullname(fullname);
     				author.setRank(Integer.valueOf(e.getAttributeValue(ValueMap.IDX_ATTRIBUTE)));
     				final String nameIdentifier = e.getAttributeValue("nameIdentifier");
     				final String nameIdentifierScheme = e.getAttributeValue("nameIdentifierScheme");
     				if (StringUtils.isNotBlank(nameIdentifier) && StringUtils.isNotBlank(nameIdentifierScheme)) {
     					author.addPid(getKV(nameIdentifierScheme, nameIdentifier));
+    				}
     				final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(fullname, false);
     				if (p.isAccurate()) {
     					author.setName(p.getNormalisedFirstName());
     					author.setSurname(p.getNormalisedSurname());
+    				}
     				metadata.addAuthor(author);
+    			}
+    		}
     		addStructuredProps(metadata, mDesc.findFieldByName("subject"), values.get("subject"), "keyword", "dnet:subject_classification_typologies");
     		addStructuredProps(metadata, mDesc.findFieldByName("title"), values.get("title"), "main title", "dnet:dataCite_title");
     		for (final String fieldname : Lists.newArrayList("description", "source", "contributor")) {
     			if (values.get(fieldname) != null) {
     				for (final String s : values.get(fieldname).listValues()) {
     					addField(metadata, mDesc.findFieldByName(fieldname), s);
+    				}
+    			}
+    		}
     		addField(metadata, mDesc.findFieldByName("language"), setQualifier(getDefaultQualifier("dnet:languages"), values.get("language").listValues()));
     		addField(metadata, mDesc.findFieldByName("dateofacceptance"), values.get("dateaccepted").listValues());
     		addField(metadata, mDesc.findFieldByName("publisher"), values.get("publisher").listValues());
     		addField(metadata, mDesc.findFieldByName("embargoenddate"), values.get("embargoenddate").listValues());
     		addField(metadata, mDesc.findFieldByName("storagedate"), values.get("storagedate").listValues());
     		String resulttype = getResultType(values);
     		addField(metadata, mDesc.findFieldByName("resulttype"), getSimpleQualifier(resulttype, "dnet:result_typologies"));
     		addField(metadata, mDesc.findFieldByName("fulltext"), values.get("fulltext").listValues());
     		addField(metadata, mDesc.findFieldByName("format"), values.get("format").listValues());
     		if (values.get("concept") != null) {
     			for (final Element e : values.get("concept")) {
     				final String id = e.getAttributes().get("id");
     				if (StringUtils.isNotBlank(id)) {
     					metadata.addContext(Context.newBuilder().setId(id));
+    				}
+    			}
+    		}
     		if (values.get("journal") != null) {
     			for (final Element e : values.get("journal")) {
     				addJournal(metadata, e);
+    			}
+    		}
     		return metadata;
+    	}
     	private static String getResultType(final ValueMap values) {
     		final Element cobjcategory = values.get("cobjcategory").stream()
     				.map(e -> StringUtils.isNotBlank(e.getText()) ? e : new Element("0000", e.getAttributes()))
     				.findFirst()
     				.orElse(new Element("0000", new HashMap<>()));
     		final String resulttype = cobjcategory.getAttributeValue("type");
     		if (StringUtils.isNotBlank(resulttype)) {
     			return resulttype;
+    		}
     		return getDefaultResulttype(cobjcategory);
+    	}
     	private static Result.Builder buildResult(final Result.Metadata.Builder metadata,
     			final ValueMap values,
     			final List<KeyValue> collectedFrom,
     			final List<KeyValue> hostedBy) {
     		final Result.Builder result = Result.newBuilder();
     		final Instance.Builder instance = Instance.newBuilder();
     		addField(instance, Instance.getDescriptor().findFieldByName("license"), values.get("license").listValues());
     		addField(instance, Instance.getDescriptor().findFieldByName("accessright"),
     				setQualifier(getDefaultQualifier("dnet:access_modes"), values.get("accessrights").listValues()));
     		addField(instance, Instance.getDescriptor().findFieldByName("instancetype"),
     				setQualifier(getDefaultQualifier("dnet:publication_resource"), values.get("cobjcategory").listValues()));
     		addField(instance, Instance.getDescriptor().findFieldByName("hostedby"), hostedBy);
     		addField(instance, Instance.getDescriptor().findFieldByName("collectedfrom"), collectedFrom);
     		addField(instance, Instance.getDescriptor().findFieldByName("dateofacceptance"), values.get("dateaccepted").listValues());
     		if (values.get("identifier") != null) {
     			addField(instance, Instance.getDescriptor().findFieldByName("url"),
     					Lists.newArrayList(Iterables.filter(values.get("identifier").listValues(), urlFilter)));
+    		}
     		result.addInstance(instance);
     		final List<Element> extrefs = values.get("reference");
     		if (!extrefs.isEmpty()) {
     			final Descriptor extDesc = ExternalReference.getDescriptor();
     			for (final Element element : extrefs) {

Project

General

Profile

D-Net

Revision 55825

Added by Claudio Atzori over 5 years ago