Revision 45981
Added by Sandro La Bruzzo about 7 years ago
| webapps/dnet-dli-container/trunk/src/main/resources/log4j.properties | ||
|---|---|---|
| 1 | 1 |
### Root Level ### |
| 2 |
log4j.rootLogger=WARN, LOGFILE |
|
| 2 |
log4j.rootLogger=WARN, LOGFILE, CONSOLE |
|
| 3 |
### Configuration for the CONSOLE appender ### |
|
| 4 |
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender |
|
| 5 |
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout |
|
| 6 |
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n |
|
| 7 |
|
|
| 8 |
|
|
| 9 |
|
|
| 3 | 10 |
### Configuration for the LOGFILE appender ### |
| 4 | 11 |
log4j.appender.LOGFILE=org.apache.log4j.RollingFileAppender |
| 5 | 12 |
log4j.appender.LOGFILE.MaxFileSize=25MB |
| modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/HadoopIsClient.java | ||
|---|---|---|
| 49 | 49 |
/** |
| 50 | 50 |
* service endpoint. |
| 51 | 51 |
*/ |
| 52 |
private Endpoint endpoint; |
|
| 53 |
private String hostname;
|
|
| 54 |
private String port;
|
|
| 52 |
// private Endpoint endpoint;
|
|
| 53 |
private String hostname;
|
|
| 54 |
private String port;
|
|
| 55 | 55 |
private String context; |
| 56 | 56 |
|
| 57 | 57 |
public String registerSetProfile(final ActionManagerSet set) throws ActionManagerException {
|
| ... | ... | |
| 330 | 330 |
this.actionManagerSetDsTemplate = actionManagerSetDsTemplate; |
| 331 | 331 |
} |
| 332 | 332 |
|
| 333 |
public Endpoint getEndpoint() {
|
|
| 334 |
return endpoint; |
|
| 335 |
} |
|
| 333 |
// public Endpoint getEndpoint() {
|
|
| 334 |
// return endpoint; |
|
| 335 |
// } |
|
| 336 |
// |
|
| 337 |
// @Required |
|
| 338 |
// public void setEndpoint(final Endpoint endpoint) {
|
|
| 339 |
// this.endpoint = endpoint; |
|
| 340 |
// } |
|
| 336 | 341 |
|
| 337 |
@Required |
|
| 338 |
public void setEndpoint(final Endpoint endpoint) {
|
|
| 339 |
this.endpoint = endpoint; |
|
| 340 |
} |
|
| 341 |
|
|
| 342 | 342 |
public String getHostname() {
|
| 343 | 343 |
return hostname; |
| 344 | 344 |
} |
| modules/dnet-hadoop-services/trunk/src/main/resources/eu/dnetlib/data/hadoop/applicationContext-dnet-hadoop-service.xml | ||
|---|---|---|
| 79 | 79 |
p:maxVersions="${services.hadoop.hbase.maxversions}"/>
|
| 80 | 80 |
|
| 81 | 81 |
<bean id="isClient" class="eu.dnetlib.data.hadoop.HadoopIsClient" |
| 82 |
p:actionManagerSetDsTemplate-ref="actionManagerSetDsTemplate"
|
|
| 83 |
p:endpoint-ref="actionManagerServiceEndpoint" p:hostname="${container.hostname}"
|
|
| 84 |
p:context="${container.context}"
|
|
| 85 |
p:port="${container.port}"/>
|
|
| 82 |
p:actionManagerSetDsTemplate-ref="actionManagerSetDsTemplate"
|
|
| 83 |
p:hostname="${container.hostname}"
|
|
| 84 |
p:context="${container.context}"
|
|
| 85 |
p:port="${container.port}"/>
|
|
| 86 | 86 |
|
| 87 | 87 |
<bean id="sequenceFileFeeder" class="eu.dnetlib.data.hadoop.blackboard.SequenceFileFeeder"/> |
| 88 | 88 |
|
| modules/dnet-data-support-services/trunk/pom.xml | ||
|---|---|---|
| 1 | 1 |
<?xml version="1.0" encoding="UTF-8"?> |
| 2 |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
| 2 |
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|
| 3 |
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
| 3 | 4 |
<parent> |
| 4 | 5 |
<groupId>eu.dnetlib</groupId> |
| 5 | 6 |
<artifactId>dnet-parent</artifactId> |
| ... | ... | |
| 38 | 39 |
<artifactId>antlr</artifactId> |
| 39 | 40 |
<groupId>antlr</groupId> |
| 40 | 41 |
</exclusion> |
| 41 |
<exclusion>
|
|
| 42 |
<artifactId>commons-lang</artifactId>
|
|
| 43 |
<groupId>commons-lang</groupId>
|
|
| 44 |
</exclusion>
|
|
| 42 |
<!--<exclusion>-->
|
|
| 43 |
<!--<artifactId>commons-lang</artifactId>-->
|
|
| 44 |
<!--<groupId>commons-lang</groupId>-->
|
|
| 45 |
<!--</exclusion>-->
|
|
| 45 | 46 |
</exclusions> |
| 46 | 47 |
</dependency> |
| 47 | 48 |
<dependency> |
| modules/dnet-dli/trunk/src/test/java/eu/dnetlib/dli/transform/DLITransformTest.java | ||
|---|---|---|
| 22 | 22 |
import org.mockito.Mock; |
| 23 | 23 |
import org.mockito.Mockito; |
| 24 | 24 |
import org.mockito.runners.MockitoJUnitRunner; |
| 25 |
import org.springframework.web.client.RestTemplate; |
|
| 25 | 26 |
|
| 26 | 27 |
import javax.xml.transform.OutputKeys; |
| 27 | 28 |
import javax.xml.transform.Transformer; |
| ... | ... | |
| 43 | 44 |
@RunWith(MockitoJUnitRunner.class) |
| 44 | 45 |
public class DLITransformTest {
|
| 45 | 46 |
|
| 47 |
|
|
| 46 | 48 |
private static final Log log = LogFactory.getLog(DLITransformTest.class); |
| 47 | 49 |
|
| 50 |
private static final String BASE_URL = "http://%s:9200/%s?pretty"; |
|
| 51 |
private static final String BASE_CFG_URL = "http://%s:9200/%s?pretty"; |
|
| 52 |
|
|
| 48 | 53 |
@Mock |
| 49 | 54 |
private DLIUtils dliUtilsInstance; |
| 50 | 55 |
|
| ... | ... | |
| 154 | 159 |
System.out.println(record); |
| 155 | 160 |
} |
| 156 | 161 |
|
| 162 |
|
|
| 157 | 163 |
} |
| modules/dnet-dli/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/DropESIndexContentJobNode.java | ||
|---|---|---|
| 3 | 3 |
import eu.dnetlib.msro.workflows.graph.Arc; |
| 4 | 4 |
import eu.dnetlib.msro.workflows.procs.Env; |
| 5 | 5 |
import org.apache.commons.io.IOUtils; |
| 6 |
|
|
| 6 |
import org.apache.commons.logging.Log; |
|
| 7 |
import org.apache.commons.logging.LogFactory; |
|
| 7 | 8 |
import org.springframework.beans.factory.annotation.Value; |
| 8 | 9 |
import org.springframework.web.client.RestTemplate; |
| 9 | 10 |
|
| 10 |
import java.net.InetAddress; |
|
| 11 |
|
|
| 12 | 11 |
/** |
| 13 | 12 |
* Created by sandro on 2/16/17. |
| 14 | 13 |
*/ |
| 15 | 14 |
|
| 16 | 15 |
|
| 17 |
class IndexSettings {
|
|
| 18 |
private int number_of_shards; |
|
| 19 |
private int number_of_replicas; |
|
| 20 | 16 |
|
| 21 |
public int getNumber_of_shards() {
|
|
| 22 |
return number_of_shards; |
|
| 23 |
} |
|
| 17 |
public class DropESIndexContentJobNode extends SimpleJobNode {
|
|
| 24 | 18 |
|
| 25 |
public void setNumber_of_shards(int number_of_shards) {
|
|
| 26 |
this.number_of_shards = number_of_shards; |
|
| 27 |
} |
|
| 19 |
private static final Log log = LogFactory.getLog(DropESIndexContentJobNode.class); |
|
| 28 | 20 |
|
| 29 |
public int getNumber_of_replicas() {
|
|
| 30 |
return number_of_replicas; |
|
| 31 |
} |
|
| 32 |
|
|
| 33 |
public void setNumber_of_replicas(int number_of_replicas) {
|
|
| 34 |
this.number_of_replicas = number_of_replicas; |
|
| 35 |
} |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
class Setting {
|
|
| 39 |
private IndexSettings settings; |
|
| 40 |
|
|
| 41 |
public IndexSettings getSettings() {
|
|
| 42 |
return settings; |
|
| 43 |
} |
|
| 44 |
|
|
| 45 |
public void setSettings(IndexSettings settings) {
|
|
| 46 |
this.settings = settings; |
|
| 47 |
} |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
|
|
| 51 |
public class DropESIndexContentJobNode extends SimpleJobNode {
|
|
| 52 |
|
|
| 53 | 21 |
private static final String BASE_URL = "http://%s:9200/%s?pretty"; |
| 54 |
private static final String BASE_CFG_URL = "http://%s:9200/%s/_mappings/%s?pretty";
|
|
| 22 |
private static final String BASE_CFG_URL = "http://%s:9200/%s?pretty"; |
|
| 55 | 23 |
@Value("${service.dli.elasticSearch.configuration}")
|
| 56 | 24 |
private String configurationIndexPath; |
| 57 | 25 |
private String index; |
| ... | ... | |
| 70 | 38 |
RestTemplate restTemplate = new RestTemplate(); |
| 71 | 39 |
restTemplate.delete(String.format(BASE_URL, host, index)); |
| 72 | 40 |
|
| 41 |
restTemplate.put(String.format(BASE_CFG_URL, host, index), config); |
|
| 73 | 42 |
|
| 74 |
Setting s = new Setting(); |
|
| 75 |
|
|
| 76 |
IndexSettings is = new IndexSettings(); |
|
| 77 |
|
|
| 78 |
is.setNumber_of_replicas(0); |
|
| 79 |
is.setNumber_of_shards(3); |
|
| 80 |
|
|
| 81 |
s.setSettings(is); |
|
| 82 |
|
|
| 83 |
restTemplate.put(String.format(BASE_URL, host, index), s); |
|
| 84 |
|
|
| 85 |
restTemplate.put(String.format(BASE_CFG_URL, host, index, type), config); |
|
| 86 |
|
|
| 87 |
System.out.println("Index Created");
|
|
| 88 |
|
|
| 89 |
|
|
| 90 | 43 |
return Arc.DEFAULT_ARC; |
| 91 | 44 |
} |
| 92 | 45 |
|
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/index/config.json | ||
|---|---|---|
| 1 | 1 |
{
|
| 2 |
"properties": {
|
|
| 3 |
"date": {
|
|
| 4 |
"type": "string", |
|
| 5 |
"index": "not_analyzed" |
|
| 6 |
}, |
|
| 7 |
"localIdentifier": {
|
|
| 2 |
"settings": {
|
|
| 3 |
"index": {
|
|
| 4 |
"number_of_shards": "3", |
|
| 5 |
"number_of_replicas": "0", |
|
| 6 |
"analysis": {
|
|
| 7 |
"analyzer": {
|
|
| 8 |
"analyzer_keyword": {
|
|
| 9 |
"tokenizer": "keyword", |
|
| 10 |
"filter": "lowercase" |
|
| 11 |
} |
|
| 12 |
} |
|
| 13 |
} |
|
| 14 |
} |
|
| 15 |
}, |
|
| 16 |
"mappings": {
|
|
| 17 |
"object": {
|
|
| 8 | 18 |
"properties": {
|
| 9 |
"id": {
|
|
| 10 |
"type": "text", |
|
| 11 |
"index": "not_analyzed" |
|
| 19 |
"date": {
|
|
| 20 |
"type": "keyword" |
|
| 12 | 21 |
}, |
| 13 |
"type": {
|
|
| 14 |
"id": {
|
|
| 15 |
"type": "text", |
|
| 16 |
"index": "not_analyzed" |
|
| 22 |
"localIdentifier": {
|
|
| 23 |
"type": "nested", |
|
| 24 |
"properties": {
|
|
| 25 |
"id": {
|
|
| 26 |
"type": "keyword" |
|
| 27 |
}, |
|
| 28 |
"type": {
|
|
| 29 |
"type": "keyword" |
|
| 30 |
} |
|
| 17 | 31 |
} |
| 18 | 32 |
} |
| 19 | 33 |
} |
| modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.dli.global.xml | ||
|---|---|---|
| 38 | 38 |
<PARAM name="reuseUnresolved" description="reuse unresolved objects on HDFS?" required="true" type="boolean" managedBy="user"/> |
| 39 | 39 |
<PARAM name="hdfsPathUnresolved" description="hdfs path for unresolved objects" required="true" type="string" managedBy="system">/tmp/scholix.dli.seq</PARAM> |
| 40 | 40 |
|
| 41 |
<PARAM name="mappingSimilarities" description="xslt mapping function for similarities" required="true" |
|
| 42 |
type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
|
|
| 43 |
<PARAM name="mappingDissimilarities" description="xslt mapping function for dissimilarities" |
|
| 44 |
required="true" type="string" managedBy="user" |
|
| 45 |
function="obtainValues('dbmf2hbaseMappings', {})"/>
|
|
| 46 |
|
|
| 47 |
|
|
| 41 | 48 |
<PARAM name="dedupConfigDataset" description="dedup configuration orchestration name" required="true" |
| 42 | 49 |
type="string" function="obtainValues('dedupOrchestrations', {})" managedBy="user"/>
|
| 43 | 50 |
<PARAM name="dedupConfigPublication" description="dedup configuration orchestration name" |
| ... | ... | |
| 45 | 52 |
managedBy="user"/> |
| 46 | 53 |
<PARAM name="dedupConfigUnknown" description="dedup configuration orchestration name" required="true" |
| 47 | 54 |
type="string" function="obtainValues('dedupOrchestrations', {})" managedBy="user"/>
|
| 48 |
<PARAM name="minDistWorkDir" description="work directory for the minDist algorithm" required="true" type="string" managedBy="user">/tmp/dedup/prod/mindist</PARAM> |
|
| 55 |
<PARAM name="minDistWorkDir" description="work directory for the minDist algorithm" required="true" |
|
| 56 |
type="string" managedBy="user">/user/dnet/dli/dedup/prod/mindist |
|
| 57 |
</PARAM> |
|
| 49 | 58 |
</PARAMETERS> |
| 50 | 59 |
<WORKFLOW> |
| 51 | 60 |
|
| modules/dnet-dli-domain/trunk/src/main/resources/eu/dnetlib/data/proto/dli/DLIObject.proto | ||
|---|---|---|
| 85 | 85 |
repeated CollectedFromType datasources = 13; |
| 86 | 86 |
|
| 87 | 87 |
} |
| 88 |
|
|
| 89 |
message ScholixIdentifier {
|
|
| 90 |
required string identifier = 1; |
|
| 91 |
required string schema = 2; |
|
| 92 |
} |
|
| 93 |
|
|
| 94 |
message ScholixEntityId {
|
|
| 95 |
required string name = 1; |
|
| 96 |
repeated ScholixIdentifier identifiers = 2; |
|
| 97 |
} |
|
| 98 |
|
|
| 99 |
message ScholixRelationship {
|
|
| 100 |
required string name = 1; |
|
| 101 |
optional string schema = 2; |
|
| 102 |
optional string inverse = 3; |
|
| 103 |
} |
|
| 104 |
|
|
| 105 |
|
|
| 106 |
message ScholixResource {
|
|
| 107 |
repeated ScholixIdentifier identifier = 1; |
|
| 108 |
required string objectType = 2; |
|
| 109 |
required string objectSubType = 3; |
|
| 110 |
required string title = 4; |
|
| 111 |
repeated ScholixEntityId creator = 5; |
|
| 112 |
optional string publicationDate = 6; |
|
| 113 |
optional ScholixEntityId publisher = 7; |
|
| 114 |
} |
|
| 115 |
|
|
| 116 |
message Scholix {
|
|
| 117 |
optional string puvlicationDate = 1; |
|
| 118 |
optional ScholixEntityId publisher = 2; |
|
| 119 |
optional ScholixEntityId linkprovider = 3; |
|
| 120 |
required ScholixRelationship relationship = 4; |
|
| 121 |
required ScholixResource source = 5; |
|
| 122 |
required ScholixResource target = 6; |
|
| 123 |
} |
|
| modules/dnet-dli-domain/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/OntologyDSResources/OntologyDSResourceType/unknown_unknown_relations.xml | ||
|---|---|---|
| 1 |
<RESOURCE_PROFILE> |
|
| 2 |
<HEADER> |
|
| 3 |
<RESOURCE_IDENTIFIER |
|
| 4 |
value="80a9a6be-b4d3-4f3d-8860-b7812da3aa38_T250b2xvZ3lEU1Jlc291cmNlcy9PbnRvbG9neURTUmVzb3VyY2VUeXBl"/> |
|
| 5 |
<RESOURCE_TYPE value="OntologyDSResourceType"/> |
|
| 6 |
<RESOURCE_KIND value="OntologyDSResources"/> |
|
| 7 |
<RESOURCE_URI value=""/> |
|
| 8 |
<DATE_OF_CREATION value="2013-11-18T10:46:37+01:00"/> |
|
| 9 |
</HEADER> |
|
| 10 |
<BODY> |
|
| 11 |
<CONFIGURATION> |
|
| 12 |
<ONTOLOGY_NAME code="unknown_unknown">dnet:unknown_unknown_relations</ONTOLOGY_NAME> |
|
| 13 |
<ONTOLOGY_DESCRIPTION>Vocabulary of the relations between Unknown entities |
|
| 14 |
</ONTOLOGY_DESCRIPTION> |
|
| 15 |
<TERMS> |
|
| 16 |
<TERM code="merges" encoding="unknown_unknown_merges" english_name="merges" |
|
| 17 |
native_name="merges"> |
|
| 18 |
<SYNONYMS/> |
|
| 19 |
<RELATIONS> |
|
| 20 |
<RELATION type="inverseOf" code="isMergedIn"/> |
|
| 21 |
</RELATIONS> |
|
| 22 |
</TERM> |
|
| 23 |
<TERM code="isMergedIn" encoding="unknown_unknown_isMergedIn" english_name="is Merged In" |
|
| 24 |
native_name="is Merged In"> |
|
| 25 |
<SYNONYMS/> |
|
| 26 |
<RELATIONS> |
|
| 27 |
<RELATION type="inverseOf" code="merges"/> |
|
| 28 |
</RELATIONS> |
|
| 29 |
</TERM> |
|
| 30 |
|
|
| 31 |
</TERMS> |
|
| 32 |
</CONFIGURATION> |
|
| 33 |
<STATUS> |
|
| 34 |
<LAST_UPDATE value="2013-11-18T10:46:36Z"/> |
|
| 35 |
</STATUS> |
|
| 36 |
<SECURITY_PARAMETERS>String</SECURITY_PARAMETERS> |
|
| 37 |
</BODY> |
|
| 38 |
</RESOURCE_PROFILE> |
|
Also available in: Unified diff