Revision 44352
Added by Sandro La Bruzzo over 7 years ago
| modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/ResolverMDStorePlugin.java | ||
|---|---|---|
| 85 | 85 |
int previousPrintValue = -1; |
| 86 | 86 |
int currentPerc = 0; |
| 87 | 87 |
|
| 88 |
for (int i = 1; i < 20; i++) {
|
|
| 89 |
|
|
| 88 |
for (int i = 1; i < 4; i++) {
|
|
| 90 | 89 |
final RecordResolver resolver = new RecordResolver(); |
| 91 | 90 |
resolver.setInputQueue(queue); |
| 92 | 91 |
resolver.setOutputCollection(resolvedRecord); |
| modules/dnet-mapreduce-jobs/trunk/src/test/java/eu/dnetlib/data/transform/XsltRowTransformerFactoryTest.java | ||
|---|---|---|
| 10 | 10 |
import com.google.common.collect.Lists; |
| 11 | 11 |
import com.google.common.collect.Maps; |
| 12 | 12 |
import com.google.common.collect.Sets; |
| 13 |
import com.google.protobuf.ExtensionRegistry; |
|
| 13 | 14 |
import com.google.protobuf.InvalidProtocolBufferException; |
| 14 | 15 |
import eu.dnetlib.data.mapreduce.hbase.index.config.*; |
| 15 | 16 |
import eu.dnetlib.data.mapreduce.util.*; |
| 17 |
import eu.dnetlib.data.proto.DNGFProtos.DNGF; |
|
| 18 |
import eu.dnetlib.data.proto.DliFieldTypeProtos; |
|
| 19 |
import eu.dnetlib.data.proto.DliProtos; |
|
| 20 |
import eu.dnetlib.data.proto.DliRels; |
|
| 16 | 21 |
import eu.dnetlib.data.proto.KindProtos.Kind; |
| 17 |
import eu.dnetlib.data.proto.DNGFProtos.DNGF; |
|
| 18 | 22 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
| 19 | 23 |
import eu.dnetlib.miscutils.functional.xml.XMLIndenter; |
| 20 | 24 |
import org.apache.commons.io.IOUtils; |
| ... | ... | |
| 166 | 170 |
} |
| 167 | 171 |
|
| 168 | 172 |
@Test |
| 169 |
public void testParseWDSDatacite() throws Exception {
|
|
| 173 |
public void testParseWDSDaaaatacite() throws Exception {
|
|
| 170 | 174 |
|
| 171 | 175 |
doTest(loadFromTransformationProfile("dmfwds2hbase.xml"), load("recordWds.xml"));
|
| 172 | 176 |
} |
| 173 | 177 |
|
| 178 |
|
|
| 174 | 179 |
@Test |
| 180 |
public void testParseDLIDatacite() throws Exception {
|
|
| 181 |
|
|
| 182 |
doTest(loadFromTransformationProfile("dmfdli2hbase.xml"), load("record_dli_dmf.xml"));
|
|
| 183 |
} |
|
| 184 |
|
|
| 185 |
@Test |
|
| 186 |
public void testParseDLIPMF() throws Exception {
|
|
| 187 |
|
|
| 188 |
final List<Row> rows = Lists.newArrayList(); |
|
| 189 |
rows.addAll(asRows(loadFromTransformationProfile("pmfdli2hbase.xml"), load("record_dli_pmf.xml")));
|
|
| 190 |
|
|
| 191 |
final ExtensionRegistry r = ExtensionRegistry.newInstance(); |
|
| 192 |
|
|
| 193 |
rows.forEach(row -> {
|
|
| 194 |
row.getColumns().forEach(result -> {
|
|
| 195 |
if (result != null) {
|
|
| 196 |
final DNGFDecoder decoder = |
|
| 197 |
DNGFDecoder.decode(result.getValue(), DliFieldTypeProtos.completionStatus, DliProtos.resolvedfrom, DliProtos.completionStatus, |
|
| 198 |
DliRels.isRelatedTo); |
|
| 199 |
|
|
| 200 |
System.out.println("decoder.getDNGF().toString() = " + decoder.getDNGF().toString());
|
|
| 201 |
} |
|
| 202 |
} |
|
| 203 |
); |
|
| 204 |
}); |
|
| 205 |
} |
|
| 206 |
|
|
| 207 |
@Test |
|
| 175 | 208 |
public void testParseDatacite() throws Exception {
|
| 176 | 209 |
|
| 177 | 210 |
doTest(loadFromTransformationProfile("dmf2hbase.xml"), load("recordDatacite.xml"));
|
| ... | ... | |
| 331 | 364 |
|
| 332 | 365 |
final Map<String, Map<String, Map<String, byte[]>>> table = buildTable(rows); |
| 333 | 366 |
|
| 334 |
// System.out.println("\n" + table.toString());
|
|
| 367 |
//System.out.println("\n" + table.toString());
|
|
| 335 | 368 |
|
| 336 | 369 |
final Map<String, XmlRecordFactory> builders = mapAll(table); |
| 337 | 370 |
|
| modules/dnet-mapreduce-jobs/trunk/src/test/resources/eu/dnetlib/data/transform/record_dli_dmf.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<record xmlns:oaf="http://namespace.dnet.eu/oaf" xmlns:oai="http://www.openarchives.org/OAI/2.0/" |
|
| 3 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri"> |
|
| 4 |
<oai:header xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> |
|
| 5 |
<dri:objIdentifier>r3d100010216::00af174e8078b3e2e929b2e35330b870</dri:objIdentifier> |
|
| 6 |
<dri:recordIdentifier>10.4121/uuid:c1d1fdbb-72df-470d-9315-d6f97e1d7c7c::http://www.onthemove-conferences.org/index.php/coopis2014</dri:recordIdentifier> |
|
| 7 |
<dri:dateOfCollection>2016-10-27T10:35:40.013+02:00</dri:dateOfCollection> |
|
| 8 |
<dri:repositoryId>3tu_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId> |
|
| 9 |
<dri:datasourceprefix>r3d100010216</dri:datasourceprefix> |
|
| 10 |
</oai:header> |
|
| 11 |
<metadata> |
|
| 12 |
<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|
| 13 |
xmlns="http://datacite.org/schema/kernel-3" |
|
| 14 |
xsi:schemaLocation="http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd"> |
|
| 15 |
<identifier xmlns="" identifierType="doi">10.4121/uuid:c1d1fdbb-72df-470d-9315-d6f97e1d7c7c</identifier> |
|
| 16 |
<creators xmlns=""> |
|
| 17 |
<creator> |
|
| 18 |
<creatorName>Munoz-Gama, J. (Jorge)</creatorName> |
|
| 19 |
</creator> |
|
| 20 |
</creators> |
|
| 21 |
<titles xmlns=""> |
|
| 22 |
<title>Large Bank Transaction Process</title> |
|
| 23 |
</titles> |
|
| 24 |
<publisher xmlns="">Universitat Politècnica de Catalunya (Barcelonatech)</publisher> |
|
| 25 |
<resourceType xmlns="" resourceTypeGeneral="dataset">dataset</resourceType> |
|
| 26 |
<relatedIdentifiers xmlns=""> |
|
| 27 |
<relatedIdentifier entityType="unknown" relatedIdentifierType="url" relationType="unknown">http://www.onthemove-conferences.org/index.php/coopis2014 |
|
| 28 |
</relatedIdentifier> |
|
| 29 |
</relatedIdentifiers> |
|
| 30 |
</resource> |
|
| 31 |
</metadata> |
|
| 32 |
<oaf:about> |
|
| 33 |
<oaf:datainfo> |
|
| 34 |
<oaf:completionStatus>complete</oaf:completionStatus> |
|
| 35 |
<oaf:collectedFrom completionStatus="incomplete" id="dli_________::r3d100010216" name="3TU.Datacentrum"/> |
|
| 36 |
<oaf:resolvedFrom completionStatus="complete" id="dli_________::datacite" name="Datasets in Datacite"/> |
|
| 37 |
</oaf:datainfo> |
|
| 38 |
</oaf:about> |
|
| 39 |
</record> |
|
| modules/dnet-mapreduce-jobs/trunk/src/test/resources/eu/dnetlib/data/transform/record_dli_pmf.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<record xmlns:oaf="http://namespace.dnet.eu/oaf" |
|
| 3 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri" |
|
| 4 |
xmlns:dc="http://purl.org/dc/elements/1.1/"> |
|
| 5 |
<oai:header xmlns:oai="http://www.openarchives.org/OAI/2.0/" |
|
| 6 |
> |
|
| 7 |
<dri:objIdentifier>elsevier____::009c77b1ea7cb5c449a599ed4f6fc7b7</dri:objIdentifier> |
|
| 8 |
<dri:recordIdentifier>10.1016/j.phytochem.2014.02.009::Z32563</dri:recordIdentifier> |
|
| 9 |
<dri:dateOfCollection>2016-10-26T18:50:23.808+02:00</dri:dateOfCollection> |
|
| 10 |
<dri:repositoryId>2d1245d4-c169-4247-9106-0f69f8d752eb_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId> |
|
| 11 |
<dri:datasourceprefix>elsevier____</dri:datasourceprefix> |
|
| 12 |
</oai:header> |
|
| 13 |
<metadata> |
|
| 14 |
<oaf:pid type="DOI">10.1016/j.phytochem.2014.02.009</oaf:pid> |
|
| 15 |
<dc:identifier>http://dx.doi.org/10.1016/j.phytochem.2014.02.009</dc:identifier> |
|
| 16 |
<dc:title>"7-Deoxyloganetic acid synthase catalyzes a key 3 step oxidation to form 7-deoxyloganetic acid in Catharanthus roseus iridoid biosynthesis"</dc:title> |
|
| 17 |
<dc:creator>Salim Vonny</dc:creator> |
|
| 18 |
<dc:creator>Wiens Brent</dc:creator> |
|
| 19 |
<dc:creator>Masada-Atsumi Sayaka</dc:creator> |
|
| 20 |
<dc:creator>Yu Fang</dc:creator> |
|
| 21 |
<dc:creator>De Luca Vincenzo</dc:creator> |
|
| 22 |
<dc:date>2014-03-01T22:15:25Z</dc:date> |
|
| 23 |
<dc:description/> |
|
| 24 |
<dc:subject>suca</dc:subject> |
|
| 25 |
<dc:subject>mela</dc:subject> |
|
| 26 |
<dc:type>publication</dc:type> |
|
| 27 |
<oaf:relatedIdentifier entityType="dataset" relatedIdentifierType="dnet" relationType="unknown">dli_resolver::53fcfde794c5ac0ea9b2542881810410</oaf:relatedIdentifier> |
|
| 28 |
<oaf:relatedIdentifier entityType="publication" relatedIdentifierType="dnet" relationType="isCitedBy">dli_resolver::100cfde794c5ac0ea9b2542881810410</oaf:relatedIdentifier> |
|
| 29 |
<oaf:relatedIdentifier entityType="unknown" relatedIdentifierType="url" relationType="unknown">http://www.google.com</oaf:relatedIdentifier> |
|
| 30 |
<dc:publisher>Elsevier BV</dc:publisher> |
|
| 31 |
</metadata> |
|
| 32 |
<oaf:about> |
|
| 33 |
<oaf:datainfo> |
|
| 34 |
<oaf:completionStatus>complete</oaf:completionStatus> |
|
| 35 |
|
|
| 36 |
<oaf:collectedFrom completionStatus="incomplete" id="dli_________::elsevier" name="Elsevier"/> |
|
| 37 |
|
|
| 38 |
<oaf:resolvedFrom completionStatus="complete" id="dli_________::crossref" name="Crossref"/> |
|
| 39 |
|
|
| 40 |
</oaf:datainfo> |
|
| 41 |
</oaf:about> |
|
| 42 |
|
|
| 43 |
</record> |
|
| modules/dnet-mapreduce-jobs/trunk/pom.xml | ||
|---|---|---|
| 59 | 59 |
<artifactId>dnet-wds-domain</artifactId> |
| 60 | 60 |
<version>1.0.0-SNAPSHOT</version> |
| 61 | 61 |
</dependency> |
| 62 |
|
|
| 62 | 63 |
<dependency> |
| 64 |
<groupId>eu.dnetlib</groupId> |
|
| 65 |
<artifactId>dnet-dli-domain</artifactId> |
|
| 66 |
<version>1.0.0-SNAPSHOT</version> |
|
| 67 |
</dependency> |
|
| 68 |
<dependency> |
|
| 63 | 69 |
<groupId>junit</groupId> |
| 64 | 70 |
<artifactId>junit</artifactId> |
| 65 | 71 |
<version>${junit.version}</version>
|
| modules/dnet-graph-domain/trunk/src/main/java/eu/dnetlib/data/mapreduce/util/DNGFRowKeyDecoder.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mapreduce.util; |
| 2 | 2 |
|
| 3 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
|
| 3 | 4 |
import org.apache.commons.lang3.StringUtils; |
| 4 | 5 |
import org.apache.commons.logging.Log; |
| 5 | 6 |
import org.apache.commons.logging.LogFactory; |
| 6 | 7 |
|
| 7 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
|
| 8 |
|
|
| 9 | 8 |
public class DNGFRowKeyDecoder {
|
| 10 | 9 |
|
| 10 |
public final static String ID_REGEX = "^[0-9][0-9]\\|.{12}::[a-zA-Z0-9]{32}$";
|
|
| 11 | 11 |
/** |
| 12 | 12 |
* logger. |
| 13 | 13 |
*/ |
| 14 | 14 |
private static final Log log = LogFactory.getLog(DNGFRowKeyDecoder.class); // NOPMD by marko on 11/24/08 5:02 PM |
| 15 |
|
|
| 16 | 15 |
private static final String SEPARATOR = "|"; |
| 17 |
|
|
| 18 |
public final static String ID_REGEX = "^[0-9][0-9]\\|.{12}::[a-zA-Z0-9]{32}$";
|
|
| 19 |
|
|
| 20 | 16 |
private String key; |
| 21 | 17 |
|
| 22 | 18 |
private Type type = null; |
| 23 | 19 |
|
| 24 | 20 |
private String id = null; |
| 25 | 21 |
|
| 26 |
public static DNGFRowKeyDecoder decode(final byte[] key) throws IllegalArgumentException {
|
|
| 27 |
return new DNGFRowKeyDecoder(new String(key)); |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public static DNGFRowKeyDecoder decode(final String key) throws IllegalArgumentException {
|
|
| 31 |
return new DNGFRowKeyDecoder(key); |
|
| 32 |
} |
|
| 33 |
|
|
| 34 | 22 |
private DNGFRowKeyDecoder(final String key) throws IllegalArgumentException {
|
| 35 | 23 |
this.key = key; |
| 36 | 24 |
|
| ... | ... | |
| 47 | 35 |
// System.out.println(OafRowTypeDecoder.class.getName() +" decoded key: " + split); |
| 48 | 36 |
} |
| 49 | 37 |
|
| 38 |
public static DNGFRowKeyDecoder decode(final byte[] key) throws IllegalArgumentException {
|
|
| 39 |
return new DNGFRowKeyDecoder(new String(key)); |
|
| 40 |
} |
|
| 41 |
|
|
| 42 |
public static DNGFRowKeyDecoder decode(final String key) throws IllegalArgumentException {
|
|
| 43 |
return new DNGFRowKeyDecoder(key); |
|
| 44 |
} |
|
| 45 |
|
|
| 50 | 46 |
public String getKey() {
|
| 51 | 47 |
return key; |
| 52 | 48 |
} |
| ... | ... | |
| 58 | 54 |
public String getId() {
|
| 59 | 55 |
return id; |
| 60 | 56 |
} |
| 61 |
} |
|
| 57 |
} |
|
| modules/dnet-graph-domain/trunk/src/main/java/eu/dnetlib/data/transform/xml/OafToHbaseXsltFunctions.java | ||
|---|---|---|
| 88 | 88 |
return entity; |
| 89 | 89 |
} |
| 90 | 90 |
|
| 91 |
private static Publication.Metadata.Builder buildMetadata(final ValueMap values, final Descriptor mDesc) {
|
|
| 91 |
protected static Publication.Metadata.Builder buildMetadata(final ValueMap values, final Descriptor mDesc) {
|
|
| 92 | 92 |
final Publication.Metadata.Builder metadata = Publication.Metadata.newBuilder(); |
| 93 | 93 |
addStructuredProps(metadata, mDesc.findFieldByName("subject"), values.get("subject").listValues(), "keyword", "dnet:result_subject");
|
| 94 | 94 |
addStructuredProps(metadata, mDesc.findFieldByName("title"), values.get("title").listValues(), "main title", "dnet:dataCite_title");
|
| ... | ... | |
| 142 | 142 |
return metadata; |
| 143 | 143 |
} |
| 144 | 144 |
|
| 145 |
private static Publication.Builder buildPublication(final Publication.Metadata.Builder metadata,
|
|
| 145 |
protected static Publication.Builder buildPublication(final Publication.Metadata.Builder metadata,
|
|
| 146 | 146 |
final ValueMap values, |
| 147 | 147 |
final Descriptor mDesc, |
| 148 | 148 |
final String hostedbyId, |
| modules/dnet-graph-domain/trunk/src/main/java/eu/dnetlib/data/transform/xml/AbstractDNetXsltFunctions.java | ||
|---|---|---|
| 1253 | 1253 |
|
| 1254 | 1254 |
elem = getDirectChild(dataInfoElement, "provenanceaction"); |
| 1255 | 1255 |
final Qualifier.Builder pBuilder = Qualifier.newBuilder(); |
| 1256 |
if (elem.hasAttributes()) {
|
|
| 1256 |
if (elem != null && elem.hasAttributes()) {
|
|
| 1257 | 1257 |
final NamedNodeMap attributes = elem.getAttributes(); |
| 1258 | 1258 |
pBuilder.setClassid(getAttributeValue(attributes, "classid")); |
| 1259 | 1259 |
pBuilder.setClassname(getAttributeValue(attributes, "classname")); |
| modules/dnet-graph-domain/trunk/src/main/resources/eu/dnetlib/data/proto/RelType.proto | ||
|---|---|---|
| 32 | 32 |
outcome = 6; // resultProject |
| 33 | 33 |
similarity = 8; // resultResult |
| 34 | 34 |
isRelatedTo = 9; // publicationDataset |
| 35 |
affiliation = 12; // resultOrganizaiton
|
|
| 35 |
affiliation = 12; // resultOrganization
|
|
| 36 | 36 |
|
| 37 | 37 |
dedup = 10; // resultResult | personPerson | organizationOrganization |
| 38 | 38 |
dedupSimilarity = 11; // resultResult | personPerson | organizationOrganization |
| modules/dnet-graph-domain/trunk/src/main/resources/eu/dnetlib/data/proto/DngfParent.proto | ||
|---|---|---|
| 96 | 96 |
optional PublicationDataset publicationDataset = 24; |
| 97 | 97 |
|
| 98 | 98 |
optional OrganizationOrganization organizationOrganization = 17; |
| 99 |
repeated KeyValue collectedfrom = 25; |
|
| 99 | 100 |
|
| 100 | 101 |
} |
| modules/dnet-graph-domain/trunk/src/main/resources/eu/dnetlib/data/proto/Rels.proto | ||
|---|---|---|
| 48 | 48 |
|
| 49 | 49 |
message PublicationPublication {
|
| 50 | 50 |
|
| 51 |
extensions 100 to 199; |
|
| 51 | 52 |
//choice of the possible subtypes |
| 52 | 53 |
optional Similarity similarity = 2; |
| 53 | 54 |
optional Dedup dedup = 4; |
| ... | ... | |
| 55 | 56 |
|
| 56 | 57 |
optional Supplement supplement = 6; |
| 57 | 58 |
optional Part part = 7; |
| 59 |
|
|
| 58 | 60 |
} |
| 59 | 61 |
|
| 60 | 62 |
message PublicationDataset {
|
| modules/dnet-dli/trunk/src/test/java/eu/dnetlib/resolver/DLIResolverTest.java | ||
|---|---|---|
| 3 | 3 |
import javax.annotation.Resource; |
| 4 | 4 |
|
| 5 | 5 |
import eu.dnetlib.resolver.model.ResolvedObject; |
| 6 |
import eu.dnetlib.resolver.model.serializer.ResolverSerializer; |
|
| 6 | 7 |
import eu.dnetlib.resolver.store.ConfigurationResolverStoreTestConfig; |
| 7 | 8 |
import org.apache.commons.logging.Log; |
| 8 | 9 |
import org.apache.commons.logging.LogFactory; |
| 9 | 10 |
import org.apache.cxf.common.i18n.Exception; |
| 10 | 11 |
import org.junit.Assert; |
| 12 |
import org.junit.Before; |
|
| 11 | 13 |
import org.junit.Test; |
| 12 | 14 |
import org.junit.runner.RunWith; |
| 15 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 13 | 16 |
import org.springframework.test.context.ContextConfiguration; |
| 14 | 17 |
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
| 15 | 18 |
|
| ... | ... | |
| 40 | 43 |
@Resource(name = "dataciteResolver") |
| 41 | 44 |
private PIDResolver dataciteResolver; |
| 42 | 45 |
|
| 46 |
@Autowired |
|
| 47 |
private ResolverStore store; |
|
| 48 |
|
|
| 49 |
@Autowired |
|
| 50 |
private ResolverSerializer serializer; |
|
| 51 |
|
|
| 52 |
@Before |
|
| 53 |
public void resetResolverStore() {
|
|
| 54 |
store.checkIntegrityCollection(); |
|
| 55 |
store.getResolverCollection().drop(); |
|
| 56 |
} |
|
| 57 |
|
|
| 43 | 58 |
@Test |
| 44 | 59 |
public void andsResolverTest() throws Exception {
|
| 45 | 60 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url");
|
| ... | ... | |
| 49 | 64 |
Assert.assertNotNull(doi); |
| 50 | 65 |
Assert.assertNotNull(doi.getTitles()); |
| 51 | 66 |
Assert.assertNotNull(doi.getDatasourceProvenance()); |
| 67 |
log.info(doi.getDatasourceProvenance()); |
|
| 52 | 68 |
|
| 53 | 69 |
} |
| 54 | 70 |
|
| ... | ... | |
| 71 | 87 |
|
| 72 | 88 |
@Test |
| 73 | 89 |
public void crossRefResolverTest() throws Exception {
|
| 74 |
final ResolvedObject pubmedObject = crossrefResolver.retrievePID("10013/epic.26895.d001", "Handle");
|
|
| 90 |
final ResolvedObject pubmedObject = crossrefResolver.retrievePID("10.1109/5.771073", "Handle");
|
|
| 75 | 91 |
Assert.assertNotNull(pubmedObject); |
| 76 | 92 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
| 77 | 93 |
Assert.assertNotNull(pubmedObject.getTitles()); |
| ... | ... | |
| 83 | 99 |
Assert.assertNotNull(pubmedObject); |
| 84 | 100 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
| 85 | 101 |
Assert.assertNotNull(pubmedObject.getTitles()); |
| 102 |
|
|
| 86 | 103 |
} |
| 87 | 104 |
|
| 88 | 105 |
@Test |
| 89 | 106 |
public void dataciteResolverTest() {
|
| 90 |
final String[] dois = { "10.6085/AA/TPT006_061MXTI061R00_20020806.50.s", "10.6085/AA/TPT006_061MXTI061R00_20020806.50.1",
|
|
| 107 |
final String[] dois = { "10.4121/uuid:c1d1fdbb-72df-470d-9315-d6f97e1d7c7c", "10.6085/AA/TPT006_061MXTI061R00_20020806.50.1",
|
|
| 91 | 108 |
"10.6085/AA/TPT006_061MXTI061R00_20020806.50.3", "10013/epic.26895.d001" }; |
| 92 | 109 |
int k = 0; |
| 93 | 110 |
double average_sum = 0.0f; |
| 94 |
Assert.assertNotNull(dataciteResolver.retrievePID(dois[3], "handle")); |
|
| 111 |
final ResolvedObject object = dataciteResolver.retrievePID(dois[0], "handle"); |
|
| 112 |
Assert.assertNotNull(object); |
|
| 113 |
log.info(object); |
|
| 114 |
log.info(object.getSubjects()); |
|
| 115 |
log.info(serializer.serializeToXML(object)); |
|
| 95 | 116 |
|
| 96 |
|
|
| 97 |
while (k < 100) {
|
|
| 98 |
for (int i = 0; i < dois.length; i++) {
|
|
| 99 |
long start = System.currentTimeMillis(); |
|
| 100 |
if (i == 0) {
|
|
| 101 |
Assert.assertNull(dataciteResolver.retrievePID(dois[i], "doi")); |
|
| 102 |
} else {
|
|
| 103 |
Assert.assertNotNull(dataciteResolver.retrievePID(dois[i], "doi")); |
|
| 104 |
} |
|
| 105 |
long end = System.currentTimeMillis(); |
|
| 106 |
average_sum += end - start; |
|
| 107 |
} |
|
| 108 |
k++; |
|
| 109 |
} |
|
| 110 |
|
|
| 111 |
log.info("Average time on request 3 item 100 times " + average_sum / 100 + "ms");
|
|
| 112 | 117 |
} |
| 113 | 118 |
|
| 114 | 119 |
} |
| modules/dnet-dli/trunk/src/test/java/eu/dnetlib/resolver/ConfigurationTestConfig.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 | 4 |
|
| 5 |
import eu.dnetlib.dli.DLIUtils; |
|
| 6 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 5 | 7 |
import eu.dnetlib.resolver.model.serializer.ResolverSerializer; |
| 6 | 8 |
import net.sf.ehcache.Cache; |
| 7 | 9 |
import org.antlr.stringtemplate.StringTemplate; |
| 8 | 10 |
import org.apache.commons.io.IOUtils; |
| 11 |
import org.apache.commons.lang3.tuple.ImmutablePair; |
|
| 9 | 12 |
import org.springframework.cache.ehcache.EhCacheFactoryBean; |
| 10 | 13 |
import org.springframework.cache.ehcache.EhCacheManagerFactoryBean; |
| 11 | 14 |
import org.springframework.context.annotation.Bean; |
| ... | ... | |
| 15 | 18 |
public class ConfigurationTestConfig {
|
| 16 | 19 |
|
| 17 | 20 |
@Bean |
| 21 |
public UniqueServiceLocator serviceLocator() {
|
|
| 22 |
return null; |
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
@Bean |
|
| 26 |
public DLIUtils dliUtils() {
|
|
| 27 |
|
|
| 28 |
DLIUtils dliUtils = new DLIUtils(); |
|
| 29 |
dliUtils.datasources.put("r3d100010464", new ImmutablePair<>("ANDS ID", "ANDS datasource"));
|
|
| 30 |
|
|
| 31 |
return dliUtils; |
|
| 32 |
|
|
| 33 |
} |
|
| 34 |
|
|
| 35 |
@Bean |
|
| 18 | 36 |
public StringTemplate pmfSerializerTemplate() throws IOException {
|
| 19 | 37 |
final String template = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dli/templates/PMFXML.st"));
|
| 20 | 38 |
final StringTemplate st = new StringTemplate(template); |
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/RepositoryServiceResources/crossref.xml | ||
|---|---|---|
| 1 | 1 |
<RESOURCE_PROFILE> |
| 2 | 2 |
<HEADER> |
| 3 | 3 |
<RESOURCE_IDENTIFIER |
| 4 |
value="3tu_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU="/>
|
|
| 4 |
value="0cf05336-894e-4793-a1f0-58d9705c4e6f_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU="/>
|
|
| 5 | 5 |
<RESOURCE_TYPE value="RepositoryServiceResourceType"/> |
| 6 | 6 |
<RESOURCE_KIND value="RepositoryServiceResources"/> |
| 7 | 7 |
<RESOURCE_URI value=""/> |
| modules/dnet-dli-domain/trunk/src/main/resources/eu/dnetlib/data/proto/dli/DliRels.proto | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.proto.dli; |
|
| 2 |
|
|
| 3 |
import "Rels.proto"; |
|
| 4 |
import "RelMetadata.proto"; |
|
| 5 |
import "SubRels.proto"; |
|
| 6 |
|
|
| 7 |
option java_package = "eu.dnetlib.data.proto"; |
|
| 8 |
option java_outer_classname = "DliRels"; |
|
| 9 |
|
|
| 10 |
|
|
| 11 |
extend PublicationPublication {
|
|
| 12 |
optional IsRelatedTo isRelatedTo = 100; |
|
| 13 |
} |
|
| modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/DLIUtils.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.dli; |
| 2 | 2 |
|
| 3 |
import java.util.HashMap; |
|
| 4 |
import java.util.List; |
|
| 5 |
import java.util.Map; |
|
| 6 |
import javax.annotation.PostConstruct; |
|
| 7 |
|
|
| 3 | 8 |
import com.google.common.collect.BiMap; |
| 4 | 9 |
import com.google.common.collect.HashBiMap; |
| 10 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 5 | 11 |
import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions; |
| 12 |
import eu.dnetlib.rmi.enabling.ISLookUpException; |
|
| 13 |
import eu.dnetlib.rmi.enabling.ISLookUpService; |
|
| 6 | 14 |
import org.apache.commons.lang3.StringUtils; |
| 15 |
import org.apache.commons.lang3.tuple.ImmutablePair; |
|
| 16 |
import org.apache.commons.lang3.tuple.Pair; |
|
| 17 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 7 | 18 |
|
| 8 | 19 |
public class DLIUtils {
|
| 9 | 20 |
|
| 21 |
public final static Map<String, Pair<String, String>> datasources = new HashMap<>(); |
|
| 22 |
private static DLIUtils instance; |
|
| 10 | 23 |
private static BiMap<String, String> relations; |
| 24 |
@Autowired |
|
| 25 |
private UniqueServiceLocator serviceLocator; |
|
| 11 | 26 |
|
| 12 | 27 |
private static BiMap<String, String> getRelationMap() {
|
| 13 | 28 |
|
| ... | ... | |
| 20 | 35 |
return relations; |
| 21 | 36 |
} |
| 22 | 37 |
|
| 38 |
public static String getNameFromDataSourcePrefix(final String datasourcePrefix) throws ISLookUpException {
|
|
| 39 |
if (datasources.keySet() == null || datasources.keySet().size() == 0) {
|
|
| 40 |
generateDSMap(); |
|
| 41 |
} |
|
| 42 |
if (!datasources.containsKey(datasourcePrefix)) |
|
| 43 |
return ""; |
|
| 44 |
return datasources.get(datasourcePrefix).getRight(); |
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public static String getIdFromDataSourcePrefix(final String datasourcePrefix) throws ISLookUpException {
|
|
| 48 |
if (datasources.keySet() == null || datasources.keySet().size() == 0) {
|
|
| 49 |
generateDSMap(); |
|
| 50 |
} |
|
| 51 |
if (!datasources.containsKey(datasourcePrefix)) |
|
| 52 |
return ""; |
|
| 53 |
return datasources.get(datasourcePrefix).getLeft(); |
|
| 54 |
} |
|
| 55 |
|
|
| 56 |
public static void generateDSMap() throws ISLookUpException {
|
|
| 57 |
if (datasources.keySet() != null && datasources.keySet().size() > 0) |
|
| 58 |
return; |
|
| 59 |
|
|
| 60 |
final String query = "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') "
|
|
| 61 |
+ "return concat($x//FIELD[./key='NamespacePrefix']/value/text(),'@--@',$x//FIELD[./key='DataSourceId']/value/text(),'@--@',$x//ENGLISH_NAME )"; |
|
| 62 |
final List<String> results = instance.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query); |
|
| 63 |
datasources.clear(); |
|
| 64 |
if (results != null) |
|
| 65 |
results.forEach(it -> {
|
|
| 66 |
final String[] splitted = it.split("@--@");
|
|
| 67 |
if (splitted != null && splitted.length == 3) {
|
|
| 68 |
datasources.put(splitted[0], new ImmutablePair<>(splitted[1], splitted[2])); |
|
| 69 |
} |
|
| 70 |
}); |
|
| 71 |
} |
|
| 72 |
|
|
| 23 | 73 |
public static String fixPID(String input) {
|
| 24 | 74 |
if (input != null) {
|
| 25 | 75 |
return input.replace("http://dx.doi.org/", "").replace("http://doi.org/", "");
|
| ... | ... | |
| 44 | 94 |
throw new RuntimeException("Error pid or pidtype cannot be null");
|
| 45 | 95 |
return DnetXsltFunctions.md5(String.format("%s::%s", pid.trim(), pidtype.toLowerCase().trim()));
|
| 46 | 96 |
} |
| 97 |
|
|
| 98 |
@PostConstruct |
|
| 99 |
public void registerInstance() throws ISLookUpException {
|
|
| 100 |
instance = this; |
|
| 101 |
//DLIUtils.generateDSMap(); |
|
| 102 |
} |
|
| 103 |
|
|
| 104 |
public UniqueServiceLocator getServiceLocator() {
|
|
| 105 |
return serviceLocator; |
|
| 106 |
} |
|
| 107 |
|
|
| 108 |
public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
|
|
| 109 |
this.serviceLocator = serviceLocator; |
|
| 110 |
} |
|
| 47 | 111 |
} |
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/resolver/xslt/transformResolvedRecord.xsl | ||
|---|---|---|
| 9 | 9 |
version="2.0"> |
| 10 | 10 |
|
| 11 | 11 |
<xsl:output method="xml" indent="yes"/> |
| 12 |
|
|
| 13 |
<xsl:param name="namespacePrefix"/>
|
|
| 14 |
|
|
| 12 |
<xsl:variable name="namespacePrefix"> |
|
| 13 |
<xsl:value-of select="string('dli_resolver')"/>
|
|
| 14 |
</xsl:variable> |
|
| 15 | 15 |
<xsl:template match="/"> |
| 16 | 16 |
<xsl:choose> |
| 17 | 17 |
<xsl:when test="//datacite:resource"> |
| ... | ... | |
| 33 | 33 |
</xsl:otherwise> |
| 34 | 34 |
</xsl:choose> |
| 35 | 35 |
</xsl:template> |
| 36 |
|
|
| 37 | 36 |
<xsl:template name="dataset"> |
| 38 | 37 |
<xsl:param name="metadata"/> |
| 39 | 38 |
<xsl:variable name="datestamp" select="datetime:dateTime()"/> |
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/templates/DMFXML.st | ||
|---|---|---|
| 1 | 1 |
<metadata> |
| 2 | 2 |
<resource xmlns="http://datacite.org/schema/kernel-3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd"> |
| 3 | 3 |
<identifier identifierType="$object.pidType$">$object.pid$</identifier> |
| 4 |
<creators> |
|
| 5 |
$object.escapedXMLAuthors:{
|
|
| 6 |
<creator> |
|
| 7 |
<creatorName>$it$</creatorName> |
|
| 8 |
</creator> |
|
| 9 |
}$ |
|
| 10 |
</creators> |
|
| 11 |
<titles> |
|
| 12 |
$object.escapedXMLTitles:{
|
|
| 13 |
<title>$it$</title> |
|
| 14 |
}$ |
|
| 15 |
</titles> |
|
| 4 |
<creators> $object.escapedXMLAuthors:{<creator><creatorName>$it$</creatorName></creator>}$ </creators>
|
|
| 5 |
<titles> $object.escapedXMLTitles:{<title>$it$</title> }$ </titles>
|
|
| 16 | 6 |
$object.datasourceProvenance:{
|
| 17 |
<publisher>$it.escapedPublisher$</publisher>
|
|
| 18 |
}$
|
|
| 7 |
$if(it.showPublisher)$<publisher>$it.escapedPublisher$</publisher>$endif$ }$
|
|
| 8 |
$if(object.date)$
|
|
| 19 | 9 |
<dates> |
| 20 | 10 |
<date dateType="Collected">$object.date$</date> |
| 21 | 11 |
</dates> |
| 12 |
$endif$ |
|
| 13 |
$if(object.subjects)$ |
|
| 22 | 14 |
<subjects> |
| 23 | 15 |
$object.subjects:{
|
| 24 | 16 |
<subject subjectScheme="$it.escapedScheme$">$it.escapedTerm$</subject> |
| 25 | 17 |
}$ |
| 26 | 18 |
</subjects> |
| 19 |
$endif$ |
|
| 27 | 20 |
<resourceType resourceTypeGeneral="$object.type$">$object.type$</resourceType> |
| 21 |
$if(object.relations)$ |
|
| 28 | 22 |
<relatedIdentifiers> |
| 29 | 23 |
$object.relations:{
|
| 30 |
<relatedIdentifier relatedIdentifierType="$it.targetPID.type$" relationType="$it.relationSemantics$">$it.targetPID.escapeXMLId$</relatedIdentifier> |
|
| 24 |
<relatedIdentifier relatedIdentifierType="$it.targetPID.type$" relationType="$it.relationSemantics$" entityType="$it.targetType$">$it.targetPID.escapeXMLId$</relatedIdentifier>
|
|
| 31 | 25 |
}$ |
| 32 | 26 |
</relatedIdentifiers> |
| 27 |
$endif$ |
|
| 28 |
$if(object.description)$ |
|
| 29 |
<descriptions> |
|
| 30 |
<description >$object.escapedDescription$</description> |
|
| 31 |
</descriptions> |
|
| 32 |
$endif$ |
|
| 33 | 33 |
</resource> |
| 34 | 34 |
</metadata> |
| 35 | 35 |
<oaf:about> |
| 36 | 36 |
<oaf:datainfo> |
| 37 | 37 |
<oaf:completionStatus>$object.completionStatus$</oaf:completionStatus> |
| 38 | 38 |
$object.datasourceProvenance:{
|
| 39 |
$if(it.showPublisher)$<oaf:hostedBy id="$it.publisherId$" name="$it.escapedPublisher$"/>$endif$ |
|
| 40 | 39 |
$if(it.showDatasource)$$if(it.resolved)$<oaf:resolvedFrom id="$it.escapedDatasourceId$" name="$it.escapedDatasource$" completionStatus="$it.completionStatus$"/> |
| 41 | 40 |
$else$ <oaf:collectedFrom id="$it.escapedDatasourceId$" name="$it.escapedDatasource$" completionStatus="$it.completionStatus$"/> |
| 42 | 41 |
$endif$ |
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/elsevier_transform.xml | ||
|---|---|---|
| 12 | 12 |
<IMPORTED/> |
| 13 | 13 |
<SCRIPT> |
| 14 | 14 |
<TITLE>DLI: Elsevier to Scholix transform</TITLE> |
| 15 |
<CODE><![CDATA[ |
|
| 16 |
<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" |
|
| 17 |
xmlns:datetime="http://exslt.org/dates-and-times" xmlns:exslt="http://exslt.org/common" |
|
| 18 |
xmlns:oaf="http://namespace.dnet.eu/oaf" |
|
| 19 |
xmlns:dc="http://purl.org/dc/elements/1.1/" |
|
| 20 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri" |
|
| 21 |
exclude-result-prefixes="xsl datetime exslt"> |
|
| 22 |
<xsl:template match="/"> |
|
| 23 |
<record> |
|
| 24 |
<xsl:copy-of select="//*[local-name()='header']"/> |
|
| 15 |
<CODE><![CDATA[<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" |
|
| 16 |
xmlns:datetime="http://exslt.org/dates-and-times" xmlns:exslt="http://exslt.org/common" |
|
| 17 |
xmlns:oaf="http://namespace.dnet.eu/oaf" |
|
| 18 |
xmlns:dc="http://purl.org/dc/elements/1.1/" |
|
| 19 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri" |
|
| 20 |
xmlns:dnet="eu.dnetlib.dli.DLIUtils" |
|
| 21 |
xmlns:scholix="http://www.scholix.org" |
|
| 22 |
exclude-result-prefixes="xsl datetime exslt dnet"> |
|
| 23 |
<xsl:template match="/"> |
|
| 24 |
<record> |
|
| 25 |
<xsl:variable name="test"><xsl:value-of select="dnet:generateDSMap()"/></xsl:variable> |
|
| 26 |
<xsl:copy-of select="//*[local-name()='header']"/> |
|
| 25 | 27 |
|
| 26 |
<xsl:variable name="source" ><xsl:value-of select="//dri:datasourceprefix"/></xsl:variable>
|
|
| 27 |
<metadata>
|
|
| 28 |
<scholix:link xmlns:scholix="http://www.scholix.org" >
|
|
| 29 |
<assertion_info>
|
|
| 30 |
<source><xsl:value-of select="$source"/></source>
|
|
| 31 |
<relationType scheme="datacite">unknown</relationType>
|
|
| 32 |
</assertion_info>
|
|
| 33 |
<source>
|
|
| 34 |
<pid type="DOI"><xsl:value-of select="//column[./@name='ArticleID']"/></pid>
|
|
| 35 |
<type>publication</type>
|
|
| 36 |
</source>
|
|
| 28 |
<xsl:variable name="source" ><xsl:value-of select="//dri:datasourceprefix"/></xsl:variable>
|
|
| 29 |
<metadata>
|
|
| 30 |
<scholix:link>
|
|
| 31 |
<assertion_info>
|
|
| 32 |
<source><xsl:value-of select="$source"/></source>
|
|
| 33 |
<relationType scheme="datacite">unknown</relationType>
|
|
| 34 |
</assertion_info>
|
|
| 35 |
<source>
|
|
| 36 |
<pid type="DOI"><xsl:value-of select="//column[./@name='ArticleID']"/></pid>
|
|
| 37 |
<type>publication</type>
|
|
| 38 |
</source>
|
|
| 37 | 39 |
|
| 38 |
<target> |
|
| 39 |
<pid> |
|
| 40 |
<xsl:attribute name="type"><xsl:value-of select="//column[./@name='db']"/></xsl:attribute> |
|
| 41 |
<xsl:value-of select="//column[./@name='datasetID']"/></pid> |
|
| 42 |
<type>dataset</type> |
|
| 43 |
</target> |
|
| 44 |
</scholix:link> |
|
| 45 |
</metadata> |
|
| 46 |
<oaf:about> |
|
| 47 |
<oaf:datainfo> |
|
| 48 |
<oaf:completionStatus>incomplete</oaf:completionStatus> |
|
| 49 |
<oaf:provisionMode>collected</oaf:provisionMode> |
|
| 50 |
</oaf:datainfo> |
|
| 51 |
</oaf:about> |
|
| 52 |
</record> |
|
| 53 |
</xsl:template> |
|
| 40 |
<target> |
|
| 41 |
<pid> |
|
| 42 |
<xsl:attribute name="type"><xsl:value-of select="//column[./@name='db']"/></xsl:attribute> |
|
| 43 |
<xsl:value-of select="//column[./@name='datasetID']"/></pid> |
|
| 44 |
<type>dataset</type> |
|
| 45 |
</target> |
|
| 46 |
</scholix:link> |
|
| 47 |
</metadata> |
|
| 48 |
<oaf:about> |
|
| 49 |
<oaf:datainfo> |
|
| 50 |
<xsl:variable name="datasourcePrefix"><xsl:value-of select="//dri:datasourceprefix/text()"/> </xsl:variable> |
|
| 51 |
|
|
| 52 |
<oaf:collectedFrom completionStatus="incomplete"> |
|
| 53 |
<xsl:attribute name="id"><xsl:value-of select="dnet:getIdFromDataSourcePrefix($datasourcePrefix)"/> </xsl:attribute> |
|
| 54 |
<xsl:attribute name="name"><xsl:value-of select="dnet:getNameFromDataSourcePrefix($datasourcePrefix)"/> </xsl:attribute> |
|
| 55 |
</oaf:collectedFrom> |
|
| 56 |
|
|
| 57 |
<oaf:completionStatus>incomplete</oaf:completionStatus> |
|
| 58 |
<oaf:provisionMode>collected</oaf:provisionMode> |
|
| 59 |
</oaf:datainfo> |
|
| 60 |
</oaf:about> |
|
| 61 |
</record> |
|
| 62 |
</xsl:template> |
|
| 54 | 63 |
</xsl:stylesheet> |
| 55 | 64 |
]]></CODE> |
| 56 | 65 |
</SCRIPT> |
| modules/dnet-dli-domain/trunk/src/main/java/eu/dnetlib/data/transform/xml/PmfToHbaseXsltFunctions.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.transform.xml; |
| 2 | 2 |
|
| 3 |
import java.util.ArrayList; |
|
| 4 |
import java.util.List; |
|
| 5 |
|
|
| 6 |
import com.google.common.base.Predicates; |
|
| 7 |
import com.google.common.collect.Iterables; |
|
| 8 |
import com.google.common.collect.Lists; |
|
| 3 | 9 |
import com.google.protobuf.Descriptors.Descriptor; |
| 10 |
import com.google.protobuf.Message; |
|
| 4 | 11 |
import eu.dnetlib.data.mapreduce.util.DNGFRowKeyDecoder; |
| 5 | 12 |
import eu.dnetlib.data.proto.DNGFProtos.DNGF; |
| 6 | 13 |
import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity; |
| 7 |
import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity.Builder; |
|
| 14 |
import eu.dnetlib.data.proto.DNGFProtos.DNGFRel; |
|
| 15 |
import eu.dnetlib.data.proto.DliFieldTypeProtos; |
|
| 16 |
import eu.dnetlib.data.proto.DliProtos; |
|
| 17 |
import eu.dnetlib.data.proto.DliRels; |
|
| 18 |
import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue; |
|
| 19 |
import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty; |
|
| 8 | 20 |
import eu.dnetlib.data.proto.PublicationProtos.Publication; |
| 21 |
import eu.dnetlib.data.proto.RelMetadataProtos.RelMetadata; |
|
| 22 |
import eu.dnetlib.data.proto.RelProtos.PublicationPublication; |
|
| 23 |
import eu.dnetlib.data.proto.RelTypeProtos.RelType; |
|
| 24 |
import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; |
|
| 25 |
import eu.dnetlib.data.proto.SubRelProtos.IsRelatedTo; |
|
| 26 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
|
| 27 |
import eu.dnetlib.resolver.model.ResolvedObject; |
|
| 28 |
import org.apache.commons.lang3.StringUtils; |
|
| 29 |
import org.w3c.dom.Node; |
|
| 9 | 30 |
import org.w3c.dom.NodeList; |
| 10 | 31 |
|
| 11 | 32 |
/** |
| ... | ... | |
| 13 | 34 |
*/ |
| 14 | 35 |
public class PmfToHbaseXsltFunctions extends OafToHbaseXsltFunctions {
|
| 15 | 36 |
|
| 37 |
//./@relatedIdentifierType, ., $namespaceprefix, $about) |
|
| 38 |
public static String createEntity( |
|
| 39 |
final String pidType, |
|
| 40 |
final String pid, |
|
| 41 |
final String namespacePrefix, |
|
| 42 |
final NodeList about) {
|
|
| 43 |
try {
|
|
| 44 |
|
|
| 45 |
final ResolvedObject obj = new ResolvedObject(); |
|
| 46 |
obj.setPid(pid); |
|
| 47 |
obj.setPidType(pidType); |
|
| 48 |
final String entityId = String.format("70|%s::%s", namespacePrefix, obj.getIdentifier());
|
|
| 49 |
|
|
| 50 |
final DNGFEntity.Builder entity = parseAbout(about); |
|
| 51 |
StructuredProperty.Builder identifier = StructuredProperty.newBuilder(); |
|
| 52 |
identifier.setQualifier(getSimpleQualifier(pidType, pidType)); |
|
| 53 |
identifier.setValue(pid); |
|
| 54 |
entity.addExtension(DliProtos.typedIdentifier, identifier.build()); |
|
| 55 |
|
|
| 56 |
entity.setId(entityId); |
|
| 57 |
entity.setType(Type.unknown); |
|
| 58 |
|
|
| 59 |
final DNGF oaf = getOaf(entity, getDataInfo(about, null, null, false, false)); |
|
| 60 |
return base64(oaf.toByteArray()); |
|
| 61 |
} catch (Throwable e) {
|
|
| 62 |
e.printStackTrace(); |
|
| 63 |
return null; |
|
| 64 |
} |
|
| 65 |
} |
|
| 66 |
|
|
| 67 |
|
|
| 68 |
|
|
| 69 |
|
|
| 16 | 70 |
//$publicationId, $provenance, $trust, $about, $originalid, $dateofcollection, $dateoftransformation, $metadata |
| 17 | 71 |
public static String pmfPublication( |
| 18 | 72 |
final String resultId, |
| ... | ... | |
| 29 | 83 |
final Descriptor mDesc = Publication.Metadata.getDescriptor(); |
| 30 | 84 |
|
| 31 | 85 |
final DNGFEntity.Builder entity = parseAbout(about); |
| 86 |
addResolvedFrom(entity, about); |
|
| 32 | 87 |
|
| 33 |
//final Publication.Metadata.Builder metadata = buildMetadata(values, mDesc); |
|
| 34 |
//final Publication.Builder result = buildPublication(metadata, values, mDesc, hostedbyId, hostedbyName); |
|
| 35 |
//final DNGFEntity.Builder entity = buildOafEntity(result, entityId, metadataNodes, getKV(collectedFromId, collectedFromName), originalId); |
|
| 88 |
final Publication.Metadata.Builder metadata = buildMetadata(values, mDesc); |
|
| 89 |
final Publication.Builder publication = buildPublication(metadata, values, mDesc, "", ""); |
|
| 90 |
final List<StructuredProperty> pids = Lists.newArrayList(); |
|
| 91 |
pids.addAll(parsePids(metadataNodes)); |
|
| 92 |
entity.setType(Type.publication).setId(resultId); |
|
| 93 |
entity.setDateoftransformation(StringUtils.isBlank(dateOfTransformation) ? "" : dateOfTransformation); |
|
| 94 |
entity.setDateofcollection(StringUtils.isBlank(dateOfCollection) ? "" : dateOfCollection); |
|
| 95 |
entity.addAllPid(Iterables.filter(pids, Predicates.notNull())); |
|
| 36 | 96 |
entity.setDateofcollection(dateOfCollection) |
| 37 | 97 |
.setDateoftransformation(dateOfTransformation).setOaiprovenance(getOAIProvenance(about)); |
| 38 |
|
|
| 98 |
entity.setPublication(publication); |
|
| 39 | 99 |
final DNGF oaf = getOaf(entity, getDataInfo(about, provenance, trust, false, false)); |
| 40 | 100 |
return base64(oaf.toByteArray()); |
| 41 | 101 |
} catch (final Throwable e) {
|
| ... | ... | |
| 44 | 104 |
return null; |
| 45 | 105 |
} |
| 46 | 106 |
|
| 47 |
private static Builder parseAbout(final NodeList about) {
|
|
| 107 |
protected static List<StructuredProperty> parsePids(final NodeList nodelist) {
|
|
| 108 |
|
|
| 109 |
final List<StructuredProperty> pids = Lists.newArrayList(); |
|
| 110 |
|
|
| 111 |
for (int i = 0; i < nodelist.getLength(); i++) {
|
|
| 112 |
final Node node = nodelist.item(i); |
|
| 113 |
Node pidType = null; |
|
| 114 |
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
|
| 115 |
if (node.getLocalName().equalsIgnoreCase("pid")) {
|
|
| 116 |
pidType = node.getAttributes().getNamedItem("type");
|
|
| 117 |
|
|
| 118 |
pids.add(getStructuredProperty(node.getTextContent(), pidType.getTextContent(), getClassName(pidType.getTextContent()), "dnet:pid_types", |
|
| 119 |
"dnet:pid_types")); |
|
| 120 |
} |
|
| 121 |
} |
|
| 122 |
} |
|
| 123 |
return pids; |
|
| 124 |
} |
|
| 125 |
|
|
| 126 |
//$publicationId, |
|
| 127 |
// $datasetId, 'publicationDataset', |
|
| 128 |
// 'publicationDataset', 'isRelatedTo', $provenance, $trust, $about |
|
| 129 |
|
|
| 130 |
public static String createRel( |
|
| 131 |
final String source, |
|
| 132 |
final String target, |
|
| 133 |
final String relType, |
|
| 134 |
final String subRelType, |
|
| 135 |
final String relationSemantic, |
|
| 136 |
final String provenanceAction, |
|
| 137 |
final String trust, |
|
| 138 |
final NodeList about) {
|
|
| 139 |
|
|
| 140 |
try {
|
|
| 141 |
final String eSource = DNGFRowKeyDecoder.decode(source).getKey(); |
|
| 142 |
final String eTarget = DNGFRowKeyDecoder.decode(target).getKey(); |
|
| 143 |
|
|
| 144 |
final RelType rType = RelType.valueOf(relType); |
|
| 145 |
final SubRelType srType = SubRelType.valueOf(subRelType); |
|
| 146 |
|
|
| 147 |
final RelMetadata.Builder metadata = RelMetadata.newBuilder().setSemantics(getSimpleQualifier(relationSemantic, relationSemantic)); |
|
| 148 |
|
|
| 149 |
final Message.Builder subRel = getSubRelBuilder(metadata, srType, null); |
|
| 150 |
|
|
| 151 |
final DNGFRel.Builder rel = getDLIRelBuilder(rType, srType, getRel(eSource, eTarget, rType, srType, relType, false), subRel); |
|
| 152 |
|
|
| 153 |
final List<KeyValue.Builder> collectedFrom = getDatasourceProvenance(about, "collectedFrom"); |
|
| 154 |
|
|
| 155 |
if (collectedFrom != null) {
|
|
| 156 |
collectedFrom.forEach(it -> rel.addCollectedfrom(it)); |
|
| 157 |
} |
|
| 158 |
|
|
| 159 |
final DNGF oaf = getOaf(rel, getDataInfo(about, provenanceAction, trust, false, false)); |
|
| 160 |
return base64(oaf.toByteArray()); |
|
| 161 |
} catch (Throwable e) {
|
|
| 162 |
e.printStackTrace(System.err); |
|
| 163 |
throw new RuntimeException(e); |
|
| 164 |
} |
|
| 165 |
|
|
| 166 |
} |
|
| 167 |
|
|
| 168 |
private static DNGFRel.Builder getDLIRelBuilder(final RelType rType, final SubRelType subRelType, DNGFRel.Builder rel, final Message.Builder subRel) {
|
|
| 169 |
|
|
| 170 |
if (rType == RelType.publicationPublication) {
|
|
| 171 |
final PublicationPublication.Builder pp = PublicationPublication.newBuilder(); |
|
| 172 |
if (subRelType == SubRelType.isRelatedTo) {
|
|
| 173 |
pp.setExtension(DliRels.isRelatedTo, (IsRelatedTo) subRel.build()); |
|
| 174 |
return rel.setPublicationPublication(pp); |
|
| 175 |
} |
|
| 176 |
} |
|
| 177 |
return getRelBuilder(rType, subRelType, rel, subRel); |
|
| 178 |
} |
|
| 179 |
|
|
| 180 |
private static DNGFEntity.Builder parseAbout(final NodeList about) {
|
|
| 48 | 181 |
final DNGFEntity.Builder entity = DNGFEntity.newBuilder(); |
| 49 | 182 |
|
| 183 |
final String completionStatus = getCompletionStatus(about); |
|
| 184 |
if (completionStatus != null) {
|
|
| 185 |
entity.setExtension(DliProtos.completionStatus, "complete"); |
|
| 186 |
} |
|
| 50 | 187 |
|
| 188 |
List<KeyValue.Builder> collectedFrom = getDatasourceProvenance(about, "collectedFrom"); |
|
| 189 |
|
|
| 190 |
if (collectedFrom != null) {
|
|
| 191 |
collectedFrom.forEach(it -> entity.addCollectedfrom(it)); |
|
| 192 |
} |
|
| 193 |
|
|
| 194 |
|
|
| 51 | 195 |
return entity; |
| 52 | 196 |
} |
| 53 | 197 |
|
| 198 |
private static void addResolvedFrom(DNGFEntity.Builder entity, NodeList about) {
|
|
| 199 |
List<KeyValue.Builder> resolvedFrom = getDatasourceProvenance(about, "resolvedFrom"); |
|
| 200 |
if (resolvedFrom != null) {
|
|
| 201 |
resolvedFrom.forEach(it -> entity.addExtension(DliProtos.resolvedfrom, it.build())); |
|
| 202 |
} |
|
| 203 |
} |
|
| 204 |
|
|
| 205 |
private static List<KeyValue.Builder> getDatasourceProvenance(NodeList about, final String nodeName) {
|
|
| 206 |
Node dataInfoNode = getDataInfo(about); |
|
| 207 |
if (dataInfoNode != null && dataInfoNode.getChildNodes() != null && dataInfoNode.getChildNodes().getLength() > 0) {
|
|
| 208 |
List<KeyValue.Builder> result = new ArrayList<>(); |
|
| 209 |
|
|
| 210 |
for (int i = 0; i < dataInfoNode.getChildNodes().getLength(); i++) {
|
|
| 211 |
final Node currentNode = dataInfoNode.getChildNodes().item(i); |
|
| 212 |
if (nodeName.equals(currentNode.getLocalName())) {
|
|
| 213 |
KeyValue.Builder currentItem = KeyValue.newBuilder(); |
|
| 214 |
final Node idNode = currentNode.getAttributes().getNamedItem("id");
|
|
| 215 |
final Node nameNode = currentNode.getAttributes().getNamedItem("name");
|
|
| 216 |
final Node completionStatusNode = currentNode.getAttributes().getNamedItem("completionStatus");
|
|
| 217 |
if (idNode != null) {
|
|
| 218 |
currentItem.setKey(idNode.getTextContent()); |
|
| 219 |
} |
|
| 220 |
if (nameNode != null) {
|
|
| 221 |
currentItem.setValue(nameNode.getTextContent()); |
|
| 222 |
} |
|
| 223 |
if (completionStatusNode != null) {
|
|
| 224 |
currentItem.setExtension(DliFieldTypeProtos.completionStatus, completionStatusNode.getTextContent()); |
|
| 225 |
} |
|
| 226 |
result.add(currentItem); |
|
| 227 |
} |
|
| 228 |
} |
|
| 229 |
return result; |
|
| 230 |
} |
|
| 231 |
return null; |
|
| 232 |
} |
|
| 233 |
|
|
| 234 |
private static String getCompletionStatus(NodeList about) {
|
|
| 235 |
Node dataInfoNode = getDataInfo(about); |
|
| 236 |
if (dataInfoNode != null && dataInfoNode.getChildNodes() != null && dataInfoNode.getChildNodes().getLength() > 0) {
|
|
| 237 |
for (int i = 0; i < dataInfoNode.getChildNodes().getLength(); i++) {
|
|
| 238 |
final Node currentNode = dataInfoNode.getChildNodes().item(i); |
|
| 239 |
if ("completionStatus".equals(currentNode.getLocalName())) {
|
|
| 240 |
return currentNode.getTextContent(); |
|
| 241 |
} |
|
| 242 |
} |
|
| 243 |
} |
|
| 244 |
return null; |
|
| 245 |
} |
|
| 246 |
|
|
| 247 |
private static Node getDataInfo(final NodeList about) {
|
|
| 248 |
if (about.getLength() > 0) {
|
|
| 249 |
final NodeList aboutChilds = about.item(0).getChildNodes(); |
|
| 250 |
for (int i = 0; i < aboutChilds.getLength(); i++) {
|
|
| 251 |
final Node currentNode = aboutChilds.item(i); |
|
| 252 |
if ("datainfo".equals(currentNode.getLocalName())) {
|
|
| 253 |
return currentNode; |
|
| 254 |
} |
|
| 255 |
} |
|
| 256 |
} |
|
| 257 |
return null; |
|
| 258 |
} |
|
| 259 |
|
|
| 260 |
|
|
| 261 |
|
|
| 54 | 262 |
} |
| modules/dnet-dli-domain/trunk/pom.xml | ||
|---|---|---|
| 45 | 45 |
<artifactId>dnet-wds-domain</artifactId> |
| 46 | 46 |
<version>1.0.0-SNAPSHOT</version> |
| 47 | 47 |
</dependency> |
| 48 |
<dependency> |
|
| 49 |
<groupId>eu.dnetlib</groupId> |
|
| 50 |
<artifactId>dnet-dli</artifactId> |
|
| 51 |
<version>1.0.0-SNAPSHOT</version> |
|
| 52 |
</dependency> |
|
| 48 | 53 |
|
| 49 | 54 |
|
| 50 | 55 |
<dependency> |
| modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/index/MdFormatNotificationHandler.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.index; |
|
| 2 |
|
|
| 3 |
import java.io.StringReader; |
|
| 4 |
import java.util.List; |
|
| 5 |
import java.util.concurrent.Executor; |
|
| 6 |
import java.util.concurrent.Executors; |
|
| 7 |
|
|
| 8 |
import eu.dnetlib.utils.MetadataReference; |
|
| 9 |
import eu.dnetlib.utils.MetadataReferenceFactory; |
|
| 10 |
import eu.dnetlib.clients.index.utils.ServiceTools; |
|
| 11 |
import eu.dnetlib.enabling.actions.AbstractSubscriptionAction; |
|
| 12 |
import org.apache.commons.logging.Log; |
|
| 13 |
import org.apache.commons.logging.LogFactory; |
|
| 14 |
import org.dom4j.Document; |
|
| 15 |
import org.dom4j.DocumentException; |
|
| 16 |
import org.dom4j.Element; |
|
| 17 |
import org.dom4j.Node; |
|
| 18 |
import org.dom4j.io.SAXReader; |
|
| 19 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 20 |
|
|
| 21 |
/** |
|
| 22 |
* MdFormatNotificationHandler updates index schema in response to metadata formats modifications. |
|
| 23 |
* |
|
| 24 |
* @author claudio |
|
| 25 |
*/ |
|
| 26 |
public class MdFormatNotificationHandler extends AbstractSubscriptionAction {
|
|
| 27 |
|
|
| 28 |
/** |
|
| 29 |
* Logger. |
|
| 30 |
*/ |
|
| 31 |
private static final Log log = LogFactory.getLog(MdFormatNotificationHandler.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
| 32 |
private final transient ThreadLocal<SAXReader> domFactory = new ThreadLocal<SAXReader>() {
|
|
| 33 |
|
|
| 34 |
@Override |
|
| 35 |
protected SAXReader initialValue() {
|
|
| 36 |
return new SAXReader(); |
|
| 37 |
} |
|
| 38 |
}; |
|
| 39 |
@Autowired |
|
| 40 |
private transient MetadataReferenceFactory mdFactory; |
|
| 41 |
@Autowired |
|
| 42 |
private IndexServerDAOMap indexServerDAOMap; |
|
| 43 |
@Autowired |
|
| 44 |
private ServiceTools serviceTools; |
|
| 45 |
private Executor executor = Executors.newSingleThreadExecutor(); |
|
| 46 |
/** |
|
| 47 |
* {@inheritDoc}
|
|
| 48 |
*/ |
|
| 49 |
private boolean enabled; |
|
| 50 |
|
|
| 51 |
@Override |
|
| 52 |
public void notified(final String subscrId, final String topic, final String rsId, final String profile) {
|
|
| 53 |
if (!topic.startsWith(getTopicPrefix())) return; |
|
| 54 |
executor.execute(() -> {
|
|
| 55 |
try {
|
|
| 56 |
final Document doc = parse(profile); |
|
| 57 |
final Node formatNode = doc.selectSingleNode("//CONFIGURATION/NAME/text()");
|
|
| 58 |
|
|
| 59 |
if ((formatNode != null) && !formatNode.asXML().isEmpty()) {
|
|
| 60 |
|
|
| 61 |
final String format = formatNode.asXML(); |
|
| 62 |
final Iterable<Element> layouts = (Iterable<Element>) doc.selectNodes("//STATUS/LAYOUTS/LAYOUT");
|
|
| 63 |
|
|
| 64 |
final Node interpretationNode = doc.selectSingleNode("//CONFIGURATION/INTERPRETATION/text()");
|
|
| 65 |
final String interpretation = interpretationNode.asXML(); |
|
| 66 |
|
|
| 67 |
log.info("found a change in mdFormat: " + format);
|
|
| 68 |
for (Element element : layouts) {
|
|
| 69 |
final String layout = element.attributeValue("name");
|
|
| 70 |
|
|
| 71 |
// updates index schema based on mdFormat and layout |
|
| 72 |
final MetadataReference mdRef = mdFactory.getMetadata(format, layout, interpretation); |
|
| 73 |
final Document fields = parse(element.selectSingleNode("./FIELDS").asXML());
|
|
| 74 |
|
|
| 75 |
List<String> backends = serviceTools.getBackendIds(mdRef); |
|
| 76 |
|
|
| 77 |
if ((backends == null) || backends.isEmpty()) {
|
|
| 78 |
log.warn("There is no backendId in profiles for mdref " + mdRef);
|
|
| 79 |
} |
|
| 80 |
|
|
| 81 |
for (String backendId : backends) {
|
|
| 82 |
IndexServerDAO idxDao = indexServerDAOMap.getIndexServerDAO(backendId); |
|
| 83 |
if (idxDao == null) throw new RuntimeException("No index found for the mdformat " + mdRef);
|
|
| 84 |
log.info("Found index DAO which serve this mdFormat");
|
|
| 85 |
idxDao.updateIndexCollection(mdRef, fields); |
|
| 86 |
} |
|
| 87 |
} |
|
| 88 |
} |
|
| 89 |
log.info("Upload schema completed");
|
|
| 90 |
} catch (Exception e) {
|
|
| 91 |
throw new RuntimeException(e); // NOPMD |
|
| 92 |
} |
|
| 93 |
}); |
|
| 94 |
} |
|
| 95 |
|
|
| 96 |
/** |
|
| 97 |
* Helper method, parses an xml string. |
|
| 98 |
* |
|
| 99 |
* @param source the xml. |
|
| 100 |
* @return the parsed xml. |
|
| 101 |
* @throws DocumentException could happen |
|
| 102 |
*/ |
|
| 103 |
private Document parse(final String source) throws DocumentException {
|
|
| 104 |
return domFactory.get().read(new StringReader(source)); |
|
| 105 |
} |
|
| 106 |
|
|
| 107 |
} |
|
| modules/dnet-msro-service/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/RefreshSchemaJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.workflows.nodes.index; |
|
| 2 |
|
|
| 3 |
import java.util.List; |
|
| 4 |
|
|
| 5 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
| 6 |
import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode; |
|
| 7 |
import eu.dnetlib.msro.workflows.procs.Env; |
|
| 8 |
import eu.dnetlib.msro.workflows.procs.Token; |
|
| 9 |
import eu.dnetlib.rmi.enabling.ISLookUpService; |
|
| 10 |
import eu.dnetlib.rmi.manager.MSROException; |
|
| 11 |
import eu.dnetlib.rmi.provision.IndexService; |
|
| 12 |
|
|
| 13 |
/** |
|
| 14 |
* Created by sandro on 10/25/16. |
|
| 15 |
*/ |
|
| 16 |
public class RefreshSchemaJobNode extends BlackboardJobNode {
|
|
| 17 |
|
|
| 18 |
private String format; |
|
| 19 |
|
|
| 20 |
private String layout; |
|
| 21 |
|
|
| 22 |
private String interpretation; |
|
| 23 |
|
|
| 24 |
@Override |
|
| 25 |
protected String obtainServiceId(final Env env) {
|
|
| 26 |
return getServiceLocator().getServiceId(IndexService.class); |
|
| 27 |
} |
|
| 28 |
|
|
| 29 |
@Override |
|
| 30 |
protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
|
|
| 31 |
final String query = "for $x in collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType') "
|
|
| 32 |
+ "where $x//NAME='%s' and $x//LAYOUT/@name='%s' and $x//INTERPRETATION='%s' " |
|
| 33 |
+ "return $x//RESOURCE_IDENTIFIER/@value/string()"; |
|
| 34 |
final ISLookUpService service = getServiceLocator().getService(ISLookUpService.class); |
|
| 35 |
|
|
| 36 |
final List<String> results = service.quickSearchProfile(String.format(query, format, layout, interpretation)); |
|
| 37 |
if (results == null || results.size() != 1) throw new MSROException( |
|
| 38 |
"Error on getting mdformat profile the xquery return unexpected values, xquery " + String.format(query, format, layout, interpretation)); |
|
| 39 |
|
|
| 40 |
job.setAction("REFRESH_SCHEMA");
|
|
| 41 |
job.getParameters().put("profileId", results.get(0));
|
|
| 42 |
|
|
| 43 |
} |
|
| 44 |
|
|
| 45 |
public String getFormat() {
|
|
| 46 |
return format; |
|
| 47 |
} |
|
| 48 |
|
|
| 49 |
public void setFormat(final String format) {
|
|
| 50 |
this.format = format; |
|
| 51 |
} |
|
| 52 |
|
|
| 53 |
public String getLayout() {
|
|
| 54 |
return layout; |
|
| 55 |
} |
|
| 56 |
|
|
| 57 |
public void setLayout(final String layout) {
|
|
| 58 |
|
|
| 59 |
this.layout = layout; |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
public String getInterpretation() {
|
|
| 63 |
return interpretation; |
|
| 64 |
} |
|
| 65 |
|
|
| 66 |
public void setInterpretation(final String interpretation) {
|
|
| 67 |
this.interpretation = interpretation; |
|
| 68 |
} |
|
| 69 |
} |
|
| modules/dnet-dli/trunk/src/test/resources/eu/dnetlib/dli/transform/elsevier.xslt | ||
|---|---|---|
| 1 |
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:datetime="http://exslt.org/dates-and-times" |
|
| 2 |
xmlns:exslt="http://exslt.org/common" xmlns:oaf="http://namespace.dnet.eu/oaf" |
|
| 3 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri" |
|
| 4 |
xmlns:dnet="eu.dnetlib.dli.DLIUtils" |
|
| 5 |
xmlns:scholix="http://www.scholix.org" |
|
| 6 |
version="2.0" |
|
| 7 |
exclude-result-prefixes="xsl datetime exslt dnet"> |
|
| 8 |
<xsl:template match="/"> |
|
| 9 |
<record> |
|
| 10 |
<xsl:variable name="test"> |
|
| 11 |
<xsl:value-of select="dnet:generateDSMap()"/> |
|
| 12 |
</xsl:variable> |
|
| 13 |
<xsl:copy-of select="//*[local-name()='header']"/> |
|
| 14 |
|
|
| 15 |
<xsl:variable name="source"> |
|
| 16 |
<xsl:value-of select="//dri:datasourceprefix"/> |
|
| 17 |
</xsl:variable> |
|
| 18 |
<metadata> |
|
| 19 |
<scholix:link> |
|
| 20 |
<suca> |
|
| 21 |
<xsl:value-of select="$test"/> |
|
| 22 |
</suca> |
|
| 23 |
|
|
| 24 |
<assertion_info> |
|
| 25 |
<source> |
|
| 26 |
<xsl:value-of select="$source"/> |
|
| 27 |
</source> |
|
| 28 |
<relationType scheme="datacite">unknown</relationType> |
|
| 29 |
</assertion_info> |
|
| 30 |
<source> |
|
| 31 |
<pid type="DOI"> |
|
| 32 |
<xsl:value-of select="//column[./@name='ArticleID']"/> |
|
| 33 |
</pid> |
|
| 34 |
<type>publication</type> |
|
| 35 |
</source> |
|
| 36 |
|
|
| 37 |
<target> |
|
| 38 |
<pid> |
|
| 39 |
<xsl:attribute name="type"> |
|
| 40 |
<xsl:value-of select="//column[./@name='db']"/> |
|
| 41 |
</xsl:attribute> |
|
| 42 |
<xsl:value-of select="//column[./@name='datasetID']"/> |
|
| 43 |
</pid> |
|
| 44 |
<type>dataset</type> |
|
| 45 |
</target> |
|
| 46 |
</scholix:link> |
|
| 47 |
</metadata> |
|
| 48 |
<oaf:about> |
|
| 49 |
<oaf:datainfo> |
|
| 50 |
<xsl:variable name="datasourcePrefix"> |
|
| 51 |
<xsl:value-of select="//dri:datasourceprefix/text()"/> |
|
| 52 |
</xsl:variable> |
|
| 53 |
|
|
| 54 |
<oaf:collectedFrom completionStatus="incomplete"> |
|
| 55 |
<xsl:attribute name="id"> |
|
| 56 |
<xsl:value-of select="dnet:getIdFromDataSourcePrefix($datasourcePrefix)"/> |
|
| 57 |
</xsl:attribute> |
|
| 58 |
<xsl:attribute name="name"> |
|
| 59 |
<xsl:value-of select="dnet:getNameFromDataSourcePrefix($datasourcePrefix)"/> |
|
| 60 |
</xsl:attribute> |
|
| 61 |
</oaf:collectedFrom> |
|
| 62 |
|
|
| 63 |
<oaf:completionStatus>incomplete</oaf:completionStatus> |
|
| 64 |
<oaf:provisionMode>collected</oaf:provisionMode> |
|
| 65 |
</oaf:datainfo> |
|
| 66 |
</oaf:about> |
|
| 67 |
</record> |
|
| 68 |
</xsl:template> |
|
| 69 |
</xsl:stylesheet> |
|
| modules/dnet-dli/trunk/src/test/resources/eu/dnetlib/dli/parser/InputRecordScholix.xml | ||
|---|---|---|
| 1 | 1 |
<?xml version="1.0" encoding="UTF-8"?> |
| 2 |
<record xmlns:oaf="http://namespace.openaire.eu/oaf" |
|
| 3 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri"> |
|
| 2 |
<record xmlns:oaf="http://namespace.dnet.eu/oaf" |
|
| 3 |
xmlns:dri="http://www.driver-repository.eu/namespace/dri" |
|
| 4 |
xmlns:scholix="http://www.scholix.org" |
|
| 5 |
> |
|
| 4 | 6 |
<oai:header xmlns:oai="http://www.openarchives.org/OAI/2.0/" |
| 5 | 7 |
> |
| 6 |
<dri:objIdentifier>dli::elsevier::0000d9de7b24b7cb315852926b88e473</dri:objIdentifier>
|
|
| 7 |
<dri:recordIdentifier>10.1016/j.jmgm.2014.05.002::3CQZ</dri:recordIdentifier>
|
|
| 8 |
<dri:dateOfCollection>2016-09-29T16:23:34.316+02:00</dri:dateOfCollection>
|
|
| 8 |
<dri:objIdentifier>elsevier____::004b41b2ca97defe95834f2b4d9abe1e</dri:objIdentifier>
|
|
| 9 |
<dri:recordIdentifier>10.1016/j.resmic.2014.02.001::P37386</dri:recordIdentifier>
|
|
| 10 |
<dri:dateOfCollection>2016-10-26T13:12:38.783+02:00</dri:dateOfCollection>
|
|
| 9 | 11 |
<dri:repositoryId>2d1245d4-c169-4247-9106-0f69f8d752eb_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId> |
| 10 |
<dri:datasourceprefix>dli::elsevier</dri:datasourceprefix>
|
|
| 12 |
<dri:datasourceprefix>elsevier____</dri:datasourceprefix>
|
|
| 11 | 13 |
</oai:header> |
| 12 | 14 |
<metadata> |
| 13 |
<scholix:link xmlns:scholix="http://www.scholix.org"> |
|
| 15 |
<scholix:link> |
|
| 16 |
<suca/> |
|
| 14 | 17 |
<assertion_info> |
| 15 |
<source>dli::elsevier</source>
|
|
| 18 |
<source>elsevier____</source>
|
|
| 16 | 19 |
<relationType scheme="datacite">unknown</relationType> |
| 17 | 20 |
</assertion_info> |
| 18 | 21 |
<source> |
| 19 |
<pid type="DOI">10.1016/j.jmgm.2014.05.002</pid>
|
|
| 22 |
<pid type="DOI">10.1016/j.resmic.2014.02.001</pid>
|
|
| 20 | 23 |
<type>publication</type> |
| 21 | 24 |
</source> |
| 22 | 25 |
<target> |
| 23 |
<pid type="pdb">3CQZ</pid>
|
|
| 26 |
<pid type="ncbi-n">P37386</pid>
|
|
| 24 | 27 |
<type>dataset</type> |
| 25 | 28 |
</target> |
| 26 | 29 |
</scholix:link> |
| 27 | 30 |
</metadata> |
| 28 |
<oaf:datainfo> |
|
| 29 |
<oaf:completionStatus>incomplete</oaf:completionStatus> |
|
| 30 |
<oaf:provisionMode>collected</oaf:provisionMode> |
|
| 31 |
</oaf:datainfo> |
|
| 31 |
<oaf:about> |
|
| 32 |
<oaf:datainfo> |
|
| 33 |
<oaf:collectedFrom completionStatus="incomplete" id="dli_________::elsevier" name="Elsevier"/> |
|
| 34 |
<oaf:completionStatus>incomplete</oaf:completionStatus> |
|
| 35 |
<oaf:provisionMode>collected</oaf:provisionMode> |
|
| 36 |
</oaf:datainfo> |
|
| 37 |
</oaf:about> |
|
| 32 | 38 |
</record> |
| modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/DataciteResolver.java | ||
|---|---|---|
| 7 | 7 |
import com.google.gson.JsonElement; |
| 8 | 8 |
import com.google.gson.JsonObject; |
| 9 | 9 |
import com.google.gson.JsonParser; |
| 10 |
import eu.dnetlib.resolver.model.CompletionStatus; |
|
| 11 |
import eu.dnetlib.resolver.model.ObjectProvenance; |
|
| 12 |
import eu.dnetlib.resolver.model.ObjectType; |
|
| 13 |
import eu.dnetlib.resolver.model.ResolvedObject; |
|
| 10 |
import eu.dnetlib.resolver.model.*; |
|
| 14 | 11 |
import org.apache.commons.logging.Log; |
| 15 | 12 |
import org.apache.commons.logging.LogFactory; |
| 16 | 13 |
|
| ... | ... | |
| 20 | 17 |
|
| 21 | 18 |
public class DataciteResolver extends AbstractPIDResolver {
|
| 22 | 19 |
|
| 23 |
public static final String DATACITE_NS_PREFIX = "dli_datacite";
|
|
| 20 |
public static final String DATACITE_NS_PREFIX = "datacite____";
|
|
| 24 | 21 |
|
| 25 | 22 |
/** |
| 26 | 23 |
* The Constant log. |
| ... | ... | |
| 30 | 27 |
/** |
| 31 | 28 |
* The Constant baseURL. |
| 32 | 29 |
*/ |
| 33 |
private final static String baseURL = "http://search.datacite.org/api?wt=json&fl=doi,title,contributor,creator,publisher&q=doi:";
|
|
| 30 |
private final static String baseURL = "http://search.datacite.org/api?wt=json&q=doi:"; |
|
| 34 | 31 |
|
| 35 | 32 |
private ResolvedObject parseResponse(final String response) {
|
| 36 | 33 |
if (response == null) return null; |
| ... | ... | |
| 54 | 51 |
responseObj.setType(ObjectType.dataset); |
| 55 | 52 |
List<String> authors = Lists.newArrayList(); |
| 56 | 53 |
List<String> titles = Lists.newArrayList(); |
| 54 |
List<SubjectType> subjects = Lists.newArrayList(); |
|
| 57 | 55 |
|
| 56 |
// ADDING SUBJECTS |
|
| 57 |
final JsonElement subjectElement = ((JsonObject) elem).get("subject");
|
|
| 58 |
if (subjectElement != null) {
|
|
| 59 |
JsonArray subjectArray = subjectElement.getAsJsonArray(); |
|
| 60 |
for (int i = 0; i < subjectArray.size(); i++) {
|
|
| 61 |
subjects.add(new SubjectType("unknown", subjectArray.get(i).getAsString()));
|
|
| 62 |
} |
|
| 63 |
} |
|
| 64 |
responseObj.setSubjects(subjects); |
|
| 65 |
|
|
| 58 | 66 |
// ADDING TITLES |
| 59 | 67 |
JsonElement titleElement = ((JsonObject) elem).get("title");
|
| 60 | 68 |
if (titleElement != null) {
|
| ... | ... | |
| 64 | 72 |
} |
| 65 | 73 |
} |
| 66 | 74 |
responseObj.setTitles(titles); |
| 75 |
|
|
| 76 |
//ADDING DESCRIPTION |
|
| 77 |
|
|
| 78 |
JsonElement descriptionElement = ((JsonObject) elem).get("description");
|
|
| 79 |
if (descriptionElement != null) {
|
|
| 80 |
JsonArray descriptionArray = descriptionElement.getAsJsonArray(); |
|
| 81 |
if (descriptionArray.size() > 0) |
|
| 82 |
responseObj.setDescription(descriptionArray.get(0).getAsString()); |
|
| 83 |
} |
|
| 84 |
|
|
| 67 | 85 |
// ADDING AUTHORS |
| 68 | 86 |
JsonElement contributorElement = ((JsonObject) elem).get("contributor");
|
| 69 | 87 |
if (contributorElement != null) {
|
| ... | ... | |
| 82 | 100 |
} |
| 83 | 101 |
responseObj.setAuthors(authors); |
| 84 | 102 |
|
| 85 |
ObjectProvenance provenance = new ObjectProvenance(DATACITE_NS_PREFIX, "resolved", "complete", null, null, true); |
|
| 103 |
ObjectProvenance provenance = new ObjectProvenance(); |
|
| 104 |
setDatasourceProvenace(provenance, DATACITE_NS_PREFIX); |
|
| 86 | 105 |
|
| 87 | 106 |
JsonElement publisherElement = ((JsonObject) elem).get("publisher");
|
| 88 | 107 |
if (publisherElement != null && !publisherElement.isJsonNull()) {
|
| modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/RCSBParser.java | ||
|---|---|---|
| 11 | 11 |
import javax.xml.stream.XMLStreamReader; |
| 12 | 12 |
|
| 13 | 13 |
import com.google.common.collect.Lists; |
| 14 |
import eu.dnetlib.resolver.model.*; |
|
| 14 |
import eu.dnetlib.resolver.model.CompletionStatus; |
|
| 15 |
import eu.dnetlib.resolver.model.ObjectProvenance; |
|
| 16 |
import eu.dnetlib.resolver.model.ObjectType; |
|
| 17 |
import eu.dnetlib.resolver.model.ResolvedObject; |
|
| 15 | 18 |
import org.apache.commons.logging.Log; |
| 16 | 19 |
import org.apache.commons.logging.LogFactory; |
| 17 | 20 |
|
| ... | ... | |
| 87 | 90 |
} |
| 88 | 91 |
if (object.getPid() != null && object.getPidType() != null) {
|
| 89 | 92 |
ObjectProvenance provenance = new ObjectProvenance(); |
| 90 |
provenance.setCompletionStatus(CompletionStatus.complete.toString()); |
|
| 91 |
provenance.setProvisionMode(ObjectProvisionMode.resolved.toString()); |
|
| 92 |
provenance.setDatasource("RCSB Protein Data Bank");
|
|
| 93 |
provenance.setDatasourceId(RCBS_NS_PREFIX); |
|
| 93 |
AbstractPIDResolver.setDatasourceProvenace(provenance, RCBS_NS_PREFIX); |
|
| 94 | 94 |
object.setDatasourceProvenance(Lists.newArrayList(provenance)); |
| 95 | 95 |
object.fixContribution(provenance); |
| 96 | 96 |
object.setType(ObjectType.dataset); |
Also available in: Unified diff
fixed bug