Revision 56978
Added by Michele Artini over 4 years ago
| modules/dnet-isti/trunk/.factorypath | ||
|---|---|---|
| 1 | 1 |
<factorypath> |
| 2 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-msro-service/5.0.0-SNAPSHOT/dnet-msro-service-5.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 3 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-core-services/1.0.0-SNAPSHOT/dnet-core-services-1.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 2 | 4 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/google/code/gson/gson/2.6.2/gson-2.6.2.jar" enabled="true" runInBatchMode="false"/> |
| 3 | 5 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/mongodb/mongo-java-driver/3.4.2/mongo-java-driver-3.4.2.jar" enabled="true" runInBatchMode="false"/> |
| 4 | 6 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/typesafe/config/1.3.0/config-1.3.0.jar" enabled="true" runInBatchMode="false"/> |
| 5 | 7 |
<factorypathentry kind="VARJAR" id="M2_REPO/javax/mail/mail/1.4.7/mail-1.4.7.jar" enabled="true" runInBatchMode="false"/> |
| 6 | 8 |
<factorypathentry kind="VARJAR" id="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar" enabled="true" runInBatchMode="false"/> |
| 7 | 9 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/codehaus/groovy/groovy-all/2.4.6/groovy-all-2.4.6.jar" enabled="true" runInBatchMode="false"/> |
| 10 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-core-components/1.0.0-SNAPSHOT/dnet-core-components-1.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 8 | 11 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/quartz-scheduler/quartz/2.2.2/quartz-2.2.2.jar" enabled="true" runInBatchMode="false"/> |
| 9 | 12 |
<factorypathentry kind="VARJAR" id="M2_REPO/c3p0/c3p0/0.9.1.1/c3p0-0.9.1.1.jar" enabled="true" runInBatchMode="false"/> |
| 10 | 13 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/slf4j/slf4j-api/1.7.7/slf4j-api-1.7.7.jar" enabled="true" runInBatchMode="false"/> |
| ... | ... | |
| 24 | 27 |
<factorypathentry kind="VARJAR" id="M2_REPO/commons-io/commons-io/2.4/commons-io-2.4.jar" enabled="true" runInBatchMode="false"/> |
| 25 | 28 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/httpcomponents/httpclient/4.4.1/httpclient-4.4.1.jar" enabled="true" runInBatchMode="false"/> |
| 26 | 29 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/httpcomponents/httpcore/4.4.1/httpcore-4.4.1.jar" enabled="true" runInBatchMode="false"/> |
| 27 |
<factorypathentry kind="VARJAR" id="M2_REPO/commons-logging/commons-logging/1.2/commons-logging-1.2.jar" enabled="true" runInBatchMode="false"/> |
|
| 28 | 30 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/cxf/cxf-core/3.1.5/cxf-core-3.1.5.jar" enabled="true" runInBatchMode="false"/> |
| 29 | 31 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/codehaus/woodstox/woodstox-core-asl/4.4.1/woodstox-core-asl-4.4.1.jar" enabled="true" runInBatchMode="false"/> |
| 30 | 32 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/ws/xmlschema/xmlschema-core/2.2.1/xmlschema-core-2.2.1.jar" enabled="true" runInBatchMode="false"/> |
| ... | ... | |
| 50 | 52 |
<factorypathentry kind="VARJAR" id="M2_REPO/xml-apis/xml-apis/1.0.b2/xml-apis-1.0.b2.jar" enabled="true" runInBatchMode="false"/> |
| 51 | 53 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/ximpleware/vtd-xml/2.13.4/vtd-xml-2.13.4.jar" enabled="true" runInBatchMode="false"/> |
| 52 | 54 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/mycila/xmltool/3.3/xmltool-3.3.jar" enabled="true" runInBatchMode="false"/> |
| 53 |
<factorypathentry kind="VARJAR" id="M2_REPO/joda-time/joda-time/2.10.1/joda-time-2.10.1.jar" enabled="true" runInBatchMode="false"/>
|
|
| 55 |
<factorypathentry kind="VARJAR" id="M2_REPO/joda-time/joda-time/2.10.2/joda-time-2.10.2.jar" enabled="true" runInBatchMode="false"/>
|
|
| 54 | 56 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-test/4.2.5.RELEASE/spring-test-4.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/> |
| 55 | 57 |
<factorypathentry kind="VARJAR" id="M2_REPO/commons-net/commons-net/3.3/commons-net-3.3.jar" enabled="true" runInBatchMode="false"/> |
| 56 | 58 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/jcraft/jsch/0.1.53/jsch-0.1.53.jar" enabled="true" runInBatchMode="false"/> |
| ... | ... | |
| 61 | 63 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/sun/xsom/xsom/20110809/xsom-20110809.jar" enabled="true" runInBatchMode="false"/> |
| 62 | 64 |
<factorypathentry kind="VARJAR" id="M2_REPO/relaxngDatatype/relaxngDatatype/20020414/relaxngDatatype-20020414.jar" enabled="true" runInBatchMode="false"/> |
| 63 | 65 |
<factorypathentry kind="VARJAR" id="M2_REPO/oro/oro/2.0.8/oro-2.0.8.jar" enabled="true" runInBatchMode="false"/> |
| 66 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-modular-uis/1.0.0-SNAPSHOT/dnet-modular-uis-1.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 67 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-information-service/1.0.0-SNAPSHOT/dnet-information-service-1.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 64 | 68 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/exist-db/exist-core/3.0.RC1/exist-core-3.0.RC1.jar" enabled="true" runInBatchMode="false"/> |
| 65 | 69 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-core/2.5.0/jackson-core-2.5.0.jar" enabled="true" runInBatchMode="false"/> |
| 66 | 70 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/github/krukow/clj-ds/0.0.4/clj-ds-0.0.4.jar" enabled="true" runInBatchMode="false"/> |
| ... | ... | |
| 118 | 122 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar" enabled="true" runInBatchMode="false"/> |
| 119 | 123 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/maven/maven-model/3.2.3/maven-model-3.2.3.jar" enabled="true" runInBatchMode="false"/> |
| 120 | 124 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/codehaus/plexus/plexus-utils/3.0.17/plexus-utils-3.0.17.jar" enabled="true" runInBatchMode="false"/> |
| 125 |
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-data-provision-services/1.0.0-SNAPSHOT/dnet-data-provision-services-1.0.0-SNAPSHOT.jar" enabled="true" runInBatchMode="false"/> |
|
| 121 | 126 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/scala-lang/scala-library/2.9.2/scala-library-2.9.2.jar" enabled="true" runInBatchMode="false"/> |
| 122 | 127 |
<factorypathentry kind="VARJAR" id="M2_REPO/com/typesafe/akka/akka-actor/2.0.4/akka-actor-2.0.4.jar" enabled="true" runInBatchMode="false"/> |
| 123 | 128 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/solr/solr-solrj/5.5.0/solr-solrj-5.5.0.jar" enabled="true" runInBatchMode="false"/> |
| ... | ... | |
| 125 | 130 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar" enabled="true" runInBatchMode="false"/> |
| 126 | 131 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/codehaus/woodstox/stax2-api/3.1.4/stax2-api-3.1.4.jar" enabled="true" runInBatchMode="false"/> |
| 127 | 132 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/noggit/noggit/0.6/noggit-0.6.jar" enabled="true" runInBatchMode="false"/> |
| 133 |
<factorypathentry kind="VARJAR" id="M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar" enabled="true" runInBatchMode="false"/> |
|
| 134 |
<factorypathentry kind="VARJAR" id="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4.jar" enabled="true" runInBatchMode="false"/> |
|
| 128 | 135 |
<factorypathentry kind="VARJAR" id="M2_REPO/mysql/mysql-connector-java/5.1.46/mysql-connector-java-5.1.46.jar" enabled="true" runInBatchMode="false"/> |
| 129 | 136 |
<factorypathentry kind="VARJAR" id="M2_REPO/org/jbibtex/jbibtex/1.0.17/jbibtex-1.0.17.jar" enabled="true" runInBatchMode="false"/> |
| 130 | 137 |
<factorypathentry kind="VARJAR" id="M2_REPO/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar" enabled="true" runInBatchMode="false"/> |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/utils/HttpFetcher.java | ||
|---|---|---|
| 44 | 44 |
.build()); |
| 45 | 45 |
|
| 46 | 46 |
public static String fetch(final String url) throws URISyntaxException {
|
| 47 |
return fetch(new URI(url), MAX_NUMBER_OF_ATTEMPTS);
|
|
| 47 |
return fetch(new URI(url), 1);
|
|
| 48 | 48 |
} |
| 49 | 49 |
|
| 50 | 50 |
public static String fetch(final URI url) {
|
| 51 |
return fetch(url, MAX_NUMBER_OF_ATTEMPTS);
|
|
| 51 |
return fetch(url, 1);
|
|
| 52 | 52 |
} |
| 53 | 53 |
|
| 54 | 54 |
public static String fetch(final URI url, final int attempts) {
|
| 55 |
if (attempts == 0) { throw new RuntimeException("Max number of attempts reached, downloading url: " + url); }
|
|
| 55 |
if (attempts > MAX_NUMBER_OF_ATTEMPTS) {
|
|
| 56 |
log.error("Max number of attempts reached, downloading url: " + url);
|
|
| 57 |
return ""; |
|
| 58 |
} |
|
| 56 | 59 |
|
| 57 | 60 |
try {
|
| 61 |
log.debug("Downloading URL: " + url + " - try: " + attempts);
|
|
| 58 | 62 |
return (new RestTemplate(httpRequestFactory)).getForObject(url, String.class); |
| 59 |
} catch (final Exception e) {
|
|
| 63 |
} catch (final Throwable e) {
|
|
| 60 | 64 |
try {
|
| 61 |
log.error("Error downloading url: " + url + " - " + e.getMessage());
|
|
| 65 |
log.error("Error downloading url: " + url + " - try: " + attempts + " - " + e.getMessage());
|
|
| 62 | 66 |
Thread.sleep(INTERVAL_MILLIS); |
| 63 |
return fetch(url, attempts - 1);
|
|
| 67 |
return fetch(url, attempts + 1);
|
|
| 64 | 68 |
} catch (final InterruptedException e1) {
|
| 65 |
throw new RuntimeException(e1); |
|
| 69 |
log.error(e); |
|
| 70 |
return ""; |
|
| 66 | 71 |
} |
| 67 | 72 |
} |
| 68 | 73 |
} |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichCollectionPlugin.java | ||
|---|---|---|
| 48 | 48 |
colls.putAll(listOaiCollections(baseUrl + "?verb=ListSets").stream() |
| 49 | 49 |
.map(this::createCollection) |
| 50 | 50 |
.filter(Objects::nonNull) |
| 51 |
.distinct() |
|
| 51 | 52 |
.collect(Collectors.toMap(CnrCollection::getCode, o -> o))); |
| 52 | 53 |
|
| 53 | 54 |
} catch (final Exception e) {
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/AbstractIstiMDStorePlugin.java | ||
|---|---|---|
| 11 | 11 |
|
| 12 | 12 |
import eu.dnetlib.data.mdstore.modular.MDStoreFeeder; |
| 13 | 13 |
import eu.dnetlib.data.mdstore.modular.action.DoneCallback; |
| 14 |
import eu.dnetlib.data.mdstore.modular.action.FailedCallback; |
|
| 14 | 15 |
import eu.dnetlib.data.mdstore.modular.action.MDStorePlugin; |
| 15 | 16 |
import eu.dnetlib.data.mdstore.modular.connector.MDStore; |
| 16 | 17 |
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao; |
| ... | ... | |
| 36 | 37 |
private static final Log log = LogFactory.getLog(AbstractIstiMDStorePlugin.class); |
| 37 | 38 |
|
| 38 | 39 |
@Override |
| 39 |
public final void run(final MDStoreDao dao, final Map<String, String> params, final DoneCallback doneCallback) throws MDStoreServiceException {
|
|
| 40 |
public final void run(final MDStoreDao dao, final Map<String, String> params, final DoneCallback doneCallback, final FailedCallback failedCallback) {
|
|
| 40 | 41 |
|
| 41 | 42 |
if (dao instanceof MDStoreDaoImpl) {
|
| 42 | 43 |
setDao((MDStoreDaoImpl) dao); |
| 43 | 44 |
} else {
|
| 44 | 45 |
log.error("a mongo mdStore service is required");
|
| 45 |
throw new MDStoreServiceException("a mongo mdStore service is required");
|
|
| 46 |
failedCallback.call(new MDStoreServiceException("a mongo mdStore service is required"));
|
|
| 46 | 47 |
} |
| 47 | 48 |
|
| 48 | 49 |
final LocalDateTime start = LocalDateTime.now(); |
| 49 |
process(resolveStore(params, "mdId"), params); |
|
| 50 |
|
|
| 51 |
log.info("Plugin Execution Time: " + ChronoUnit.SECONDS.between(start, LocalDateTime.now()));
|
|
| 52 |
|
|
| 53 |
doneCallback.call(params); |
|
| 50 |
try {
|
|
| 51 |
process(resolveStore(params, "mdId"), params); |
|
| 52 |
log.info("Plugin Execution Time: " + ChronoUnit.SECONDS.between(start, LocalDateTime.now()));
|
|
| 53 |
doneCallback.call(params); |
|
| 54 |
} catch (final Throwable e) {
|
|
| 55 |
log.error(e.getMessage(), e); |
|
| 56 |
failedCallback.call(e); |
|
| 57 |
} |
|
| 54 | 58 |
} |
| 55 | 59 |
|
| 56 | 60 |
protected abstract void process(final MongoMDStore store, final Map<String, String> params) throws MDStoreServiceException; |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliObjectType.java | ||
|---|---|---|
| 1 | 1 |
|
| 2 | 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
| 3 | 3 |
|
| 4 |
@Deprecated |
|
| 4 | 5 |
public class DliObjectType {
|
| 5 | 6 |
|
| 6 | 7 |
private String subType; |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliEntity.java | ||
|---|---|---|
| 7 | 7 |
|
| 8 | 8 |
private List<DliIdentifier> identifiers = null; |
| 9 | 9 |
private List<DliProvider> objectProvider = null; |
| 10 |
private DliObjectType objectType;
|
|
| 10 |
private String objectType;
|
|
| 11 | 11 |
private List<DliPublisher> publisher = null; |
| 12 | 12 |
private String title; |
| 13 |
private List<DliCreator> creators = null; |
|
| 13 | 14 |
|
| 14 | 15 |
public List<DliIdentifier> getIdentifiers() {
|
| 15 | 16 |
return identifiers; |
| ... | ... | |
| 27 | 28 |
this.objectProvider = objectProvider; |
| 28 | 29 |
} |
| 29 | 30 |
|
| 30 |
public DliObjectType getObjectType() {
|
|
| 31 |
public String getObjectType() {
|
|
| 31 | 32 |
return objectType; |
| 32 | 33 |
} |
| 33 | 34 |
|
| 34 |
public void setObjectType(final DliObjectType objectType) {
|
|
| 35 |
public void setObjectType(final String objectType) {
|
|
| 35 | 36 |
this.objectType = objectType; |
| 36 | 37 |
} |
| 37 | 38 |
|
| ... | ... | |
| 51 | 52 |
this.title = title; |
| 52 | 53 |
} |
| 53 | 54 |
|
| 55 |
public List<DliCreator> getCreators() {
|
|
| 56 |
return creators; |
|
| 57 |
} |
|
| 58 |
|
|
| 59 |
public void setCreators(final List<DliCreator> creators) {
|
|
| 60 |
this.creators = creators; |
|
| 61 |
} |
|
| 62 |
|
|
| 54 | 63 |
} |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliRelation.java | ||
|---|---|---|
| 9 | 9 |
private DliRelationship relationship; |
| 10 | 10 |
private DliEntity source; |
| 11 | 11 |
private DliEntity target; |
| 12 |
private String publicationDate; |
|
| 12 | 13 |
|
| 13 | 14 |
public List<DliProvider> getLinkProvider() {
|
| 14 | 15 |
return linkProvider; |
| ... | ... | |
| 42 | 43 |
this.target = target; |
| 43 | 44 |
} |
| 44 | 45 |
|
| 46 |
public String getPublicationDate() {
|
|
| 47 |
return publicationDate; |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
public void setPublicationDate(final String publicationDate) {
|
|
| 51 |
this.publicationDate = publicationDate; |
|
| 52 |
} |
|
| 53 |
|
|
| 45 | 54 |
} |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliCreator.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 2 |
|
|
| 3 |
import java.util.List; |
|
| 4 |
|
|
| 5 |
public class DliCreator {
|
|
| 6 |
|
|
| 7 |
private String name = null; |
|
| 8 |
private List<DliIdentifier> identifiers = null; |
|
| 9 |
|
|
| 10 |
public String getName() {
|
|
| 11 |
return name; |
|
| 12 |
} |
|
| 13 |
|
|
| 14 |
public void setName(final String name) {
|
|
| 15 |
this.name = name; |
|
| 16 |
} |
|
| 17 |
|
|
| 18 |
public List<DliIdentifier> getIdentifiers() {
|
|
| 19 |
return identifiers; |
|
| 20 |
} |
|
| 21 |
|
|
| 22 |
public void setIdentifiers(final List<DliIdentifier> identifiers) {
|
|
| 23 |
this.identifiers = identifiers; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/CnrCollection.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mdstore.plugins.objects; |
| 2 | 2 |
|
| 3 |
import java.util.Objects; |
|
| 4 |
|
|
| 3 | 5 |
import javax.xml.bind.annotation.XmlAccessType; |
| 4 | 6 |
import javax.xml.bind.annotation.XmlAccessorType; |
| 5 | 7 |
import javax.xml.bind.annotation.XmlElement; |
| ... | ... | |
| 37 | 39 |
public void setName(final String name) {
|
| 38 | 40 |
this.name = name; |
| 39 | 41 |
} |
| 42 |
|
|
| 43 |
@Override |
|
| 44 |
public int hashCode() {
|
|
| 45 |
return Objects.hash(code); |
|
| 46 |
} |
|
| 47 |
|
|
| 48 |
@Override |
|
| 49 |
public boolean equals(final Object obj) {
|
|
| 50 |
if (this == obj) { return true; }
|
|
| 51 |
if (obj == null) { return false; }
|
|
| 52 |
if (!(obj instanceof CnrCollection)) { return false; }
|
|
| 53 |
final CnrCollection other = (CnrCollection) obj; |
|
| 54 |
return Objects.equals(code, other.code); |
|
| 55 |
} |
|
| 40 | 56 |
} |
Also available in: Unified diff
- failedCallback
- update of dli objects
- WorkAround for duplicated sets from people repository