Project

General

Profile

« Previous | Next » 

Revision 54287

added Crossref Event Data

View differences:

modules/dnet-dli/trunk/src/test/java/eu/dnetlib/resolver/DLIResolverTest.java
79 79

  
80 80

  
81 81
        List<String> dois = Arrays.asList(
82
                "10.4225/andstest",
83
                "10.4230/dagrep.1.2.1",
84
                "10.4230/dagrep.1.2.30",
85
                "10.1594/wdcc/clm_c20_3_d3",
86
                "10.4230/dagrep.1.2.67",
87
                "10.4230/dagrep.1.2.16",
88
                "10.4230/dagrep.1.2.i",
89
                "10.4230/dagrep.1.3.1",
90
                "10.4230/lipics.iclp.2011.162",
91
                "10.4230/lipics.iclp.2011.187",
92
                "10.4230/lipics.iclp.2011.1",
93
                "10.4230/lipics.iclp.2011.198",
94
                "10.4230/lipics.iclp.2011.209",
95
                "10.4230/lipics.iclp.2011.i",
96
                "10.4230/lipics.iclp.2011.220",
97
                "10.4230/lipics.iclp.2011.128",
98
                "10.4230/lipics.iclp.2011.11",
99
                "10.4230/lipics.iclp.2011.231",
100
                "10.4230/dagrep.1.2.47",
101
                "10.4230/lipics.iclp.2011.117");
82
                "10.1594/pangaea.230638");
102 83

  
103 84
        DataciteOfflineResolver resolver = new DataciteOfflineResolver();
104 85
        resolver.setMongoClient(client);
modules/dnet-dli/trunk/src/test/java/eu/dnetlib/dli/collector/plugin/CrossRefIteratorTest.java
1
package eu.dnetlib.dli.collector.plugin;
2

  
3
import org.antlr.stringtemplate.StringTemplate;
4
import org.apache.commons.io.IOUtils;
5
import org.junit.Test;
6

  
7
import java.io.IOException;
8
import java.io.InputStream;
9

  
10
public class CrossRefIteratorTest {
11

  
12

  
13
    @Test
14
    public void CrossRefIteratorTest() throws IOException {
15
        final InputStream resourceAsStream = getClass().getResourceAsStream("/eu/dnetlib/dli/templates/Scholixv1.st");
16

  
17

  
18
        final String template = IOUtils.toString(resourceAsStream);
19
        final StringTemplate st = new StringTemplate(template);
20

  
21
        CrossRefIterator ci = new CrossRefIterator(st);
22
        int i =0;
23
        while (ci.hasNext()){
24
            ci.next();
25

  
26
            i++;
27
            if (i % 100 ==0 ) {
28
                System.out.println("Read "+i);
29
            }
30
        }
31
    }
32
}
modules/dnet-dli/trunk/src/test/resources/eu/dnetlib/dli/templates/Scholixv1.st
1
<link xmlns="http://www.scholix.org">
2
    $if(object.date)$<publicationDate>$object.date$</publicationDate>$endif$
3
    <linkProvider>
4
        $object.datasourceProvenance:{
5
        <name>$it.datasource$</name>
6
        <identifiers>
7
            <identifier>$it.datasourceId$</identifier>
8
        </identifiers>
9
    </linkProvider>
10
    }$
11
    $object.relations:{
12
    <relashionship>
13
        <name>$it.relationSemantics$</name>
14
        <schema>datacite</schema>
15
        $if(it.inverseRelation)$
16
        <inverseRelashionship>$it.inverseRelation$</inverseRelashionship>
17
        $endif$
18
    </relashionship>
19
    }$
20
    <source>
21
        <identifier>
22
            <identifier>$object.pid$</identifier>
23
            <schema>$object.type$</schema>
24
        </identifier>
25
        <objectType>
26
            <type>$object.type$</type>
27
        </objectType>
28
    </source>
29
    <target>
30
        $object.relations:{
31
        <identifier>
32
            <identifier>$it.targetPID.id$</identifier>
33
            <schema>$it.targetPID.type$</schema>
34
        </identifier>
35
        <objectType>
36
            <type>$it.targetType$</type>
37
        </objectType>
38
        }$
39
    </target>
40
</link>
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/DLIRecordResolver.java
55 55

  
56 56
		boolean shouldUpdate = false;
57 57
		if (inputObject.getCompletionStatus() == null || !inputObject.getCompletionStatus().equals(CompletionStatus.complete.toString())) {
58
			shouldUpdate = shouldUpdate || tryToResolveRecord(inputObject);
58
			shouldUpdate = tryToResolveRecord(inputObject);
59 59
		}
60 60
        return resolveRelations(inputRecord, inputObject, shouldUpdate);
61 61

  
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/resolver/DataciteOfflineResolver.java
7 7
import com.mongodb.client.FindIterable;
8 8
import com.mongodb.client.MongoCollection;
9 9
import com.mongodb.client.model.Filters;
10
import eu.dnetlib.dli.resolver.model.CompletionStatus;
10 11
import eu.dnetlib.dli.resolver.model.DLIObjectProvenance;
11 12
import eu.dnetlib.dli.resolver.model.DLIResolvedObject;
12 13
import eu.dnetlib.enabling.tools.DnetStreamSupport;
......
16 17
import org.apache.commons.logging.Log;
17 18
import org.apache.commons.logging.LogFactory;
18 19
import org.bson.Document;
20
import org.springframework.beans.factory.annotation.Required;
19 21
import org.springframework.beans.factory.annotation.Value;
20 22

  
21 23

  
22 24
import java.util.List;
25
import java.util.Objects;
23 26
import java.util.stream.Collectors;
24 27

  
25 28

  
......
55 58
                return parseResponse(result);
56 59
            }
57 60
        } catch (Throwable e) {
58
            log.error("Error on getting item with Pid: " +  pid);
61
            log.error("Error on getting item with Pid: " +  pid, e);
59 62
        }
60 63
        return null;
61 64
    }
......
78 81
            dli.setSubjects(
79 82
                    DnetStreamSupport.generateStreamFromIterator(root.getAsJsonArray("subjects").iterator())
80 83
                    .map(JsonElement::getAsJsonObject)
81
                    .map(it -> new SubjectType(it.get("scheme").getAsString(), it.get("term").getAsString()))
84
                    .map(this::createSubjectType)
85
                    .filter(Objects::nonNull)
82 86
                    .collect(Collectors.toList())
83 87
            );
84 88
        }
85
        dli.setDatasourceProvenance(
86
                DnetStreamSupport.generateStreamFromIterator(root.getAsJsonArray("datasourceProvenance").iterator())
87
                .map(JsonElement::getAsJsonObject)
88
                .map(it -> {
89
                    final DLIObjectProvenance provenance =new DLIObjectProvenance();
90
                    provenance.setDatasourceId(it.get("datasourceId").getAsString());
91
                    provenance.setCompletionStatus(it.get("completionStatus").getAsString());
92
                    provenance.setProvisionMode(it.get("provisionMode").getAsString());
93
                    provenance.setDatasource(it.get("datasource").getAsString());
94
                    provenance.setPublisher(getStringValue(it, "publisher"));
95
                    return provenance;
96
                })
97
                .collect(Collectors.toList())
98
        );
89
        dli.setDatasourceProvenance(A);
90
        dli.setCompletionStatus(CompletionStatus.complete.toString());
99 91
        return dli;
100 92
    }
101 93

  
94
    private SubjectType createSubjectType(final JsonObject item){
95
        if (item.has("term") && !item.get("term").isJsonNull())
96
            return new SubjectType(item.get("scheme").getAsString(), item.get("term").getAsString());
97
        else {
98
            return null;
99
        }
100
    }
101

  
102 102
    private List<String> getStringValues(final JsonObject root, final String key) {
103 103

  
104 104
        if (root.has(key) && root.get(key).isJsonArray()) {
......
127 127

  
128 128
    }
129 129

  
130

  
130
    @Required
131 131
    public void setMongoClient(final MongoClient client) {
132 132
        this.client = client;
133 133
    }
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/collector/plugin/CrossRefIterator.java
64 64
        }
65 65
        if (buffer.isEmpty()) return null;
66 66
        final JsonObject item = buffer.remove(0).getAsJsonObject();
67
        final DLIResolvedObject object = new DLIResolvedObject();
68
        final String id = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
69
        final String id_type = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
70
        object.setPid(id);
71
        object.setPidType(id_type);
72
        final String type = item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
73
        object.setType(type.equals("literature")? ObjectType.publication:ObjectType.dataset);
74
        final String relType = item.get("RelationshipType").getAsJsonObject().get("Name").getAsString();
75
        object.setDatasourceProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
76
        DLIObjectRelation relation = new DLIObjectRelation();
77
        relation.setRelationSemantics(relType);
78
        final String t_id = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
79
        final String t_id_type = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
80
        final String t_type = item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
67
        try {
68
            final DLIResolvedObject object = new DLIResolvedObject();
69
            final String id = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
70
            final String id_type = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
71
            object.setPid(id);
72
            object.setPidType(id_type);
73
            final String type = item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").isJsonNull()?"unknownw":item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
74
            object.setType(type.equals("literature") ? ObjectType.publication : ObjectType.dataset);
75
            final String relType = item.get("RelationshipType").getAsJsonObject().get("Name").getAsString();
76
            object.setDatasourceProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
77
            DLIObjectRelation relation = new DLIObjectRelation();
78
            relation.setRelationSemantics(relType);
79
            final String t_id = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
80
            final String t_id_type = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
81 81

  
82
        relation.setTargetPID(new PID(t_id,t_id_type));
83
        relation.setTargetType(t_type.equals("literature")? ObjectType.publication:ObjectType.dataset);
84
        object.setRelations(Collections.singletonList(relation));
85
        relation.setCompletionStatus("incomplete");
86
        relation.setRelationProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
87
        try {
88
            StringWriter writer = new StringWriter(16);
89
            NoIndentWriter out = new NoIndentWriter(writer);
90
            resolverSerializer.removeAttribute("object");
91
            resolverSerializer.setAttribute("object", object);
92
            resolverSerializer.write(out);
93
            String result = writer.toString();
94
            writer.close();
95
            return result;
96
        } catch (Throwable e) {
82
            final String t_type = item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").isJsonNull()?"unknown":item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
83

  
84
            relation.setTargetPID(new PID(t_id, t_id_type));
85
            relation.setTargetType(t_type.equals("literature") ? ObjectType.publication : ObjectType.dataset);
86
            object.setRelations(Collections.singletonList(relation));
87
            relation.setCompletionStatus("incomplete");
88
            relation.setRelationProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
89
            try {
90
                StringWriter writer = new StringWriter(16);
91
                NoIndentWriter out = new NoIndentWriter(writer);
92
                resolverSerializer.removeAttribute("object");
93
                resolverSerializer.setAttribute("object", object);
94
                resolverSerializer.write(out);
95
                String result = writer.toString();
96
                writer.close();
97
                return result;
98
            } catch (Throwable e) {
99
                throw new RuntimeException(e);
100
            }
101
        }catch (Throwable e) {
102
            System.out.println(item.toString());
97 103
            throw new RuntimeException(e);
98 104
        }
99 105
    }
......
108 114
            if (jElement.getAsJsonObject().has("message"))
109 115
            {
110 116
                JsonObject message = jElement.getAsJsonObject().get("message").getAsJsonObject();
111
                nextCursor = message.get("next-cursor").getAsString();
117
                if (message.has("next-cursor") && !message.get("next-cursor").isJsonNull())
118
                    nextCursor = message.get("next-cursor").getAsString();
119
                else
120
                    nextCursor = null;
112 121
                total = message.get("total-results").getAsInt();
113 122
                final JsonArray items = message.get("link-packages").getAsJsonArray();
114 123
                items.forEach(buffer::add);
115
                return true;
124
                return nextCursor!=null;
116 125
            }
117 126
        }
118 127
        return false;
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/resolver/applicationContext-dli-resolver.xml
46 46
		<!--<property name="cache" ref="dliResolverCache"/>-->
47 47
	<!--</bean>-->
48 48

  
49
    <bean id="dataciteOfflineResolver" class="eu.dnetlib.dli.resolver.DataciteOfflineResolver" p:order="0" p:availableOffline="true">
49
    <bean id="dataciteOfflineResolver"
50
		  class="eu.dnetlib.dli.resolver.DataciteOfflineResolver"
51
		  p:order="0" p:availableOffline="true"
52
		  p:mongoClient-ref="mongoClient">
50 53
        <property name="cache" ref="dliResolverCache"/>
54

  
51 55
    </bean>
52 56

  
53 57
    <bean id="ncbinResolver" class="eu.dnetlib.dli.resolver.NCBINResolver" p:order="4" p:availableOffline="false">
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/templates/Scholixv1.st
19 19
    }$
20 20
    <source>
21 21
        <identifier>
22
            <identifier>$object.pid$</identifier>
23
            <schema>$object.type$</schema>
22
            <identifier>$object.escapedXMLPid$</identifier>
23
            <schema>$object.pidType$</schema>
24 24
        </identifier>
25 25
        <objectType>
26 26
            <type>$object.type$</type>
......
29 29
    <target>
30 30
        $object.relations:{
31 31
        <identifier>
32
            <identifier>$it.targetPID.id$</identifier>
32
            <identifier>$it.targetPID.escapeXMLId$</identifier>
33 33
            <schema>$it.targetPID.type$</schema>
34 34
        </identifier>
35 35
        <objectType>
modules/dnet-dli/trunk/pom.xml
10 10
	<modelVersion>4.0.0</modelVersion>
11 11
	<groupId>eu.dnetlib</groupId>
12 12
	<artifactId>dnet-dli</artifactId>
13
	<packaging>pom</packaging>
13
	<packaging>jar</packaging>
14 14
	<version>1.0.0-SNAPSHOT</version>
15

  
16

  
15 17
	<scm>
16 18
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-dli/trunk</developerConnection>
17 19
	</scm>

Also available in: Unified diff