Revision 48014
Added by Claudio Atzori almost 7 years ago
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/deploy.info | ||
---|---|---|
1 |
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-actionmanager-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-actionmanager-service"} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/java/eu/dnetlib/actionmanager/xslt/RelationXsltFunctionsTest.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.xslt; |
|
2 |
|
|
3 |
import java.io.StringReader; |
|
4 |
import java.io.StringWriter; |
|
5 |
|
|
6 |
import javax.xml.transform.Source; |
|
7 |
import javax.xml.transform.Transformer; |
|
8 |
import javax.xml.transform.TransformerFactory; |
|
9 |
import javax.xml.transform.stream.StreamResult; |
|
10 |
import javax.xml.transform.stream.StreamSource; |
|
11 |
|
|
12 |
import org.junit.Before; |
|
13 |
import org.junit.Test; |
|
14 |
|
|
15 |
public class RelationXsltFunctionsTest { |
|
16 |
private Transformer transformer; |
|
17 |
|
|
18 |
@Before |
|
19 |
public void setUp() throws Exception { |
|
20 |
Source source = new StreamSource(getClass().getResourceAsStream("/eu/dnetlib/actionmanager/xslt/rels2actions.xslt")); |
|
21 |
transformer = TransformerFactory.newInstance().newTransformer(source); |
|
22 |
} |
|
23 |
|
|
24 |
@Test |
|
25 |
public void test_01() throws Exception { |
|
26 |
StringBuffer xml = new StringBuffer(); |
|
27 |
xml.append("<RELATIONS>"); |
|
28 |
xml.append("</RELATIONS>"); |
|
29 |
|
|
30 |
applyXslt(xml.toString()); |
|
31 |
} |
|
32 |
|
|
33 |
@Test |
|
34 |
public void test_02() throws Exception { |
|
35 |
StringBuffer xml = new StringBuffer(); |
|
36 |
xml.append("<RELATIONS>"); |
|
37 |
xml.append("<RELATION type='resultProject' source='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' target='corda_______::281877' />"); |
|
38 |
xml.append("<RELATION type='resultProject' source='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' target='40|corda_______::0001fc1603869410927ddbde3afb2403' />"); |
|
39 |
xml.append("<RELATION type='personResult' source='30|crossref____::000178abc5a314472603976a1be68927' target='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' />"); |
|
40 |
xml.append("</RELATIONS>"); |
|
41 |
|
|
42 |
applyXslt(xml.toString()); |
|
43 |
} |
|
44 |
|
|
45 |
public String applyXslt(String xml) throws Exception { |
|
46 |
final Source xmlSource = new StreamSource(new StringReader(xml)); |
|
47 |
final StringWriter output = new StringWriter(); |
|
48 |
|
|
49 |
transformer.transform(xmlSource, new StreamResult(output)); |
|
50 |
|
|
51 |
System.out.println(output.toString()); |
|
52 |
|
|
53 |
return output.toString(); |
|
54 |
} |
|
55 |
|
|
56 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/resources/eu/dnetlib/actionmanager/test/applicationContext-actionmanager-test.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<beans xmlns="http://www.springframework.org/schema/beans" |
|
3 |
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws" |
|
4 |
xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing" |
|
5 |
xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration" |
|
6 |
xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template" |
|
7 |
xmlns:util="http://www.springframework.org/schema/util" |
|
8 |
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd |
|
9 |
http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd |
|
10 |
http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd |
|
11 |
http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd |
|
12 |
http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd |
|
13 |
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd |
|
14 |
http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd"> |
|
15 |
|
|
16 |
<bean id="hbaseActionManagerCore" class="eu.dnetlib.actionmanager.hbase.HBaseActionManagerCore" |
|
17 |
p:actionFactory-ref="actionFactory" |
|
18 |
p:hbaseClient-ref="hbaseClient" /> |
|
19 |
|
|
20 |
<bean id="hbaseClient" class="eu.dnetlib.actionmanager.hbase.HBaseClient" |
|
21 |
p:tableName="db_actions_michele" |
|
22 |
p:hbaseConf="${services.data.hbase.dm.properties}" |
|
23 |
init-method="initWithEmptyTable" /> |
|
24 |
<!-- p:hbaseConf="classpath:/eu/dnetlib/actionmanager/test/hbase.properties" --> |
|
25 |
|
|
26 |
<bean id="actionFactory" class="eu.dnetlib.actionmanager.actions.ActionFactory"> |
|
27 |
<property name="xslts"> |
|
28 |
<util:map> |
|
29 |
<entry key="dmf2actions" value="classpath:/eu/dnetlib/actionmanager/xslt/dmf2actions.xslt" /> |
|
30 |
</util:map> |
|
31 |
</property> |
|
32 |
</bean> |
|
33 |
|
|
34 |
|
|
35 |
</beans> |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/resources/eu/dnetlib/actionmanager/test/hbase.properties | ||
---|---|---|
1 |
#COMMON |
|
2 |
hadoop.entry.node = node1.hadoop.research-infrastructures.eu |
|
3 |
|
|
4 |
#CORE-SITE |
|
5 |
fs.defaultFS = hdfs://${hadoop.entry.node}:8020 |
|
6 |
io.file.buffer.size = 65536 |
|
7 |
io.compression.codecs = org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec |
|
8 |
hadoop.security.authentication = simple |
|
9 |
hadoop.security.auth_to_local = DEFAULT |
|
10 |
|
|
11 |
#HBASE-SITE |
|
12 |
hbase.rootdir = ${fs.defaultFS}/hbase |
|
13 |
hbase.client.write.buffer = 2097152 |
|
14 |
hbase.client.pause = 1000 |
|
15 |
hbase.client.retries.number = 10 |
|
16 |
hbase.client.scanner.caching = 1 |
|
17 |
hbase.client.keyvalue.maxsize = 10485760 |
|
18 |
hbase.security.authentication = simple |
|
19 |
zookeeper.session.timeout = 60000 |
|
20 |
zookeeper.znode.parent = /hbase |
|
21 |
zookeeper.znode.rootserver = root-region-server |
|
22 |
hbase.zookeeper.quorum = ${hadoop.entry.node} |
|
23 |
hbase.zookeeper.property.clientPort = 2182 |
|
24 |
|
|
25 |
#HDFS-SITE |
|
26 |
dfs.https.port = 50470 |
|
27 |
dfs.namenode.https-address = ${hadoop.entry.node}:${dfs.https.port} |
|
28 |
dfs.namenode.http-address = ${hadoop.entry.node}:50070 |
|
29 |
dfs.replication = 3 |
|
30 |
dfs.blocksize = 134217728 |
|
31 |
dfs.client.use.datanode.hostname= false |
|
32 |
|
|
33 |
#MAPRED-SITE |
|
34 |
mapreduce.jobtracker.address = ${hadoop.entry.node}:8021 |
|
35 |
mapred.job.tracker = ${hadoop.entry.node}:8021 |
|
36 |
mapred.mapper.new-api = true |
|
37 |
mapred.reducer.new-api = true |
|
38 |
mapred.output.compress = false |
|
39 |
mapred.output.compression.type = BLOCK |
|
40 |
mapred.output.compression.codec = org.apache.hadoop.io.compress.DefaultCodec |
|
41 |
#mapred.map.output.compression.codec = org.apache.hadoop.io.compress.SnappyCodec |
|
42 |
mapred.compress.map.output = true |
|
43 |
#io.sort.factor = 64 |
|
44 |
#io.sort.record.percent = 0.05 |
|
45 |
#io.sort.spill.percent = 0.8 |
|
46 |
#mapred.reduce.parallel.copies = 10 |
|
47 |
#mapred.submit.replication = 1 |
|
48 |
#mapred.reduce.tasks = 8 |
|
49 |
#io.sort.mb = 56 |
|
50 |
#mapred.child.java.opts = -Xmx238449165 |
|
51 |
#mapred.job.reuse.jvm.num.tasks = 1 |
|
52 |
#mapred.map.tasks = 6 |
|
53 |
#mapred.map.tasks.speculative.execution = false |
|
54 |
#mapred.reduce.tasks.speculative.execution = false |
|
55 |
#mapred.reduce.slowstart.completed.maps = 0.8 |
|
56 |
##mapred.local.dir = /data/mapred/local,/data/mapred/jt |
|
57 |
mapred.local.dir = /tmp |
|
58 |
mapred.child.java.opts = -Xmx512973352 |
|
59 |
|
|
60 |
#dfs.client.use.legacy.blockreader=true |
|
61 |
|
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/BlackboardJobListener.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import eu.dnetlib.actionmanager.rmi.ActionManagerException; |
|
4 |
import eu.dnetlib.enabling.tools.blackboard.AbstractBlackboardJobListener; |
|
5 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
6 |
import org.apache.commons.logging.Log; |
|
7 |
import org.apache.commons.logging.LogFactory; |
|
8 |
|
|
9 |
public class BlackboardJobListener extends AbstractBlackboardJobListener { |
|
10 |
|
|
11 |
private static final Log log = LogFactory.getLog(BlackboardJobListener.class); |
|
12 |
|
|
13 |
private JobCallback callback; |
|
14 |
|
|
15 |
public BlackboardJobListener(final JobCallback callback) { |
|
16 |
super(); |
|
17 |
this.callback = callback; |
|
18 |
} |
|
19 |
|
|
20 |
@Override |
|
21 |
protected void onDone(final BlackboardJob job) { |
|
22 |
try { |
|
23 |
if (callback != null) { |
|
24 |
callback.onDone(job.getParameters()); |
|
25 |
} |
|
26 |
} catch (Exception e) { |
|
27 |
log.error("fail to run a callback function"); |
|
28 |
} |
|
29 |
} |
|
30 |
|
|
31 |
@Override |
|
32 |
protected void onFailed(final BlackboardJob job) { |
|
33 |
try { |
|
34 |
if (callback != null) { |
|
35 |
callback.onFailed(new ActionManagerException(job.getError())); |
|
36 |
} |
|
37 |
} catch (Exception e) { |
|
38 |
log.error("fail to run a callback function"); |
|
39 |
} |
|
40 |
} |
|
41 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/PromoteActionManagerActionFromHDFS.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
import java.util.Set; |
|
5 |
|
|
6 |
import com.google.common.base.Splitter; |
|
7 |
import com.google.common.collect.Sets; |
|
8 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
9 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction; |
|
10 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler; |
|
11 |
import org.apache.commons.lang.StringUtils; |
|
12 |
|
|
13 |
public class PromoteActionManagerActionFromHDFS extends AbstractActionManagerAction implements BlackboardServerAction<ActionManagerActions> { |
|
14 |
|
|
15 |
@Override |
|
16 |
public void execute(final BlackboardServerHandler handler, final BlackboardJob job) { |
|
17 |
|
|
18 |
final String setParam = job.getParameters().get("set"); |
|
19 |
|
|
20 |
Set<String> sets = null; |
|
21 |
if (StringUtils.isNotBlank(setParam) && !setParam.equalsIgnoreCase(JobLauncher.ALL_SETS)) { |
|
22 |
sets = Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(setParam)); |
|
23 |
} |
|
24 |
|
|
25 |
try { |
|
26 |
getHbaseJobLauncher().executePromoteFromHDFSJob(sets, new JobCallback() { |
|
27 |
|
|
28 |
@Override |
|
29 |
public void onDone(final Map<String, String> properties) { |
|
30 |
job.getParameters().putAll(properties); |
|
31 |
handler.done(job); |
|
32 |
} |
|
33 |
|
|
34 |
@Override |
|
35 |
public void onFailed(final Throwable e) { |
|
36 |
handler.failed(job, e); |
|
37 |
} |
|
38 |
|
|
39 |
}); |
|
40 |
} catch (Throwable e) { |
|
41 |
handler.failed(job, e); |
|
42 |
} |
|
43 |
|
|
44 |
} |
|
45 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/JobCallback.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
|
|
5 |
public interface JobCallback { |
|
6 |
|
|
7 |
void onDone(Map<String, String> properties); |
|
8 |
|
|
9 |
void onFailed(Throwable e); |
|
10 |
|
|
11 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/GarbageActionManagerActionFromHDFS.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
4 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction; |
|
5 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler; |
|
6 |
|
|
7 |
public class GarbageActionManagerActionFromHDFS extends AbstractActionManagerAction implements BlackboardServerAction<ActionManagerActions> { |
|
8 |
|
|
9 |
@Override |
|
10 |
public void execute(final BlackboardServerHandler handler, final BlackboardJob job) { |
|
11 |
try { |
|
12 |
getHbaseJobLauncher().executeGarbageActions(); |
|
13 |
handler.done(job); |
|
14 |
} catch (Throwable e) { |
|
15 |
handler.failed(job, e); |
|
16 |
} |
|
17 |
} |
|
18 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/ActionManagerActions.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
public enum ActionManagerActions { |
|
4 |
PROMOTE_FROM_HDFS, GARBAGE_HDFS |
|
5 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/AbstractActionManagerAction.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import javax.annotation.Resource; |
|
4 |
|
|
5 |
public class AbstractActionManagerAction { |
|
6 |
|
|
7 |
@Resource |
|
8 |
private JobLauncher jobLauncher; |
|
9 |
|
|
10 |
public JobLauncher getHbaseJobLauncher() { |
|
11 |
return jobLauncher; |
|
12 |
} |
|
13 |
|
|
14 |
public void setJobLauncher(final JobLauncher jobLauncher) { |
|
15 |
this.jobLauncher = jobLauncher; |
|
16 |
} |
|
17 |
|
|
18 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/JobLauncher.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.blackboard; |
|
2 |
|
|
3 |
import java.text.ParseException; |
|
4 |
import java.util.Date; |
|
5 |
import java.util.List; |
|
6 |
import java.util.Map; |
|
7 |
import java.util.Set; |
|
8 |
import java.util.concurrent.Executor; |
|
9 |
import java.util.concurrent.Executors; |
|
10 |
import javax.annotation.Resource; |
|
11 |
|
|
12 |
import com.google.common.base.Function; |
|
13 |
import com.google.common.base.Joiner; |
|
14 |
import com.google.common.base.Predicate; |
|
15 |
import com.google.common.collect.Iterables; |
|
16 |
import com.google.common.collect.Lists; |
|
17 |
import com.google.common.collect.Maps; |
|
18 |
import eu.dnetlib.actionmanager.is.ISClient; |
|
19 |
import eu.dnetlib.actionmanager.rmi.ActionManagerException; |
|
20 |
import eu.dnetlib.actionmanager.set.ActionManagerSet; |
|
21 |
import eu.dnetlib.actionmanager.set.RawSet; |
|
22 |
import eu.dnetlib.data.hadoop.config.ClusterName; |
|
23 |
import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions; |
|
24 |
import eu.dnetlib.data.hadoop.rmi.HadoopService; |
|
25 |
import eu.dnetlib.data.hadoop.rmi.HadoopServiceException; |
|
26 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
|
27 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
28 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; |
|
29 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
30 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
31 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardClientHandler; |
|
32 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
33 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJobRegistry; |
|
34 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
35 |
import org.apache.commons.logging.Log; |
|
36 |
import org.apache.commons.logging.LogFactory; |
|
37 |
import org.springframework.beans.factory.annotation.Autowired; |
|
38 |
import org.springframework.beans.factory.annotation.Required; |
|
39 |
import org.springframework.beans.factory.annotation.Value; |
|
40 |
|
|
41 |
public class JobLauncher { |
|
42 |
|
|
43 |
public static final String ALL_SETS = "__ALL__"; |
|
44 |
public static final String SEMICOLON = ";"; |
|
45 |
public static final String COLON = ","; |
|
46 |
public static final String SEQFILE_INPUTFORMAT = "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"; |
|
47 |
private final static String UPDATE_ACTION_PROFILE = "for $x in " |
|
48 |
+ "/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'ActionManagerSetDSResourceType' and .//SET/@id = '%s'] " |
|
49 |
+ "return update delete $x//RAW_SETS/EXPIRED[@id = '%s']"; |
|
50 |
private static final Log log = LogFactory.getLog(JobLauncher.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
51 |
private final Executor executor = Executors.newCachedThreadPool(); |
|
52 |
|
|
53 |
@Resource |
|
54 |
private UniqueServiceLocator serviceLocator; |
|
55 |
|
|
56 |
private ISClient infomationServiceClient; |
|
57 |
|
|
58 |
/** |
|
59 |
* blackboard handler. |
|
60 |
*/ |
|
61 |
@Resource |
|
62 |
private BlackboardClientHandler blackboardClientHandler; |
|
63 |
|
|
64 |
/** |
|
65 |
* blackboard job registry. |
|
66 |
*/ |
|
67 |
@Resource |
|
68 |
private BlackboardJobRegistry jobRegistry; |
|
69 |
|
|
70 |
@Autowired |
|
71 |
private ISClient isClient; |
|
72 |
|
|
73 |
@Value("${services.actionmanager.promote.mapper.class}") |
|
74 |
private String promoteJobMapperClass; |
|
75 |
|
|
76 |
private String actionTable; |
|
77 |
private String dataTable; |
|
78 |
|
|
79 |
public void executePromoteFromHDFSJob(final Set<String> sets, final JobCallback callback) throws ActionManagerException, ISLookUpException { |
|
80 |
log.info("Starting commit m/r job; sets=" + sets); |
|
81 |
|
|
82 |
final List<String> paths = Lists.newArrayList(); |
|
83 |
if (sets == null || sets.isEmpty()) { |
|
84 |
|
|
85 |
for (ActionManagerSet set : infomationServiceClient.listSets()) { |
|
86 |
paths.add(set.getPathToLatest()); |
|
87 |
} |
|
88 |
|
|
89 |
} else { |
|
90 |
for (String setId : sets) { |
|
91 |
if (infomationServiceClient.existsSet(setId)) { |
|
92 |
final ActionManagerSet set = infomationServiceClient.getSet(setId); |
|
93 |
paths.add(set.getPathToLatest()); |
|
94 |
} else { |
|
95 |
log.error("Invalid set " + setId); |
|
96 |
throw new ActionManagerException("Invalid set " + setId); |
|
97 |
} |
|
98 |
} |
|
99 |
} |
|
100 |
|
|
101 |
final HadoopService hadoopService = serviceLocator.getService(HadoopService.class); |
|
102 |
final List<String> existingPaths = Lists.newArrayList(Iterables.filter(paths, new Predicate<String>(){ |
|
103 |
@Override |
|
104 |
public boolean apply(final String path) { |
|
105 |
|
|
106 |
try { |
|
107 |
final boolean exist = hadoopService.existHdfsPath(ClusterName.DM.toString(), path); |
|
108 |
if (!exist) { |
|
109 |
log.warn(String.format("path '%s' doesn't exist on DM cluster'", path)); |
|
110 |
} |
|
111 |
return exist; |
|
112 |
} catch (HadoopServiceException e) { |
|
113 |
log.error(e); |
|
114 |
return false; |
|
115 |
} |
|
116 |
} |
|
117 |
})); |
|
118 |
|
|
119 |
final Map<String, String> params = Maps.newHashMap(); |
|
120 |
params.put("mapred.input.dir.formats", Joiner.on(COLON).join(Iterables.transform(existingPaths, new Function<String, String>() { |
|
121 |
@Override |
|
122 |
public String apply(final String path) { |
|
123 |
return path + SEMICOLON + SEQFILE_INPUTFORMAT; |
|
124 |
} |
|
125 |
}))); |
|
126 |
params.put("mapred.input.dir.mappers", Joiner.on(COLON).join(Iterables.transform(existingPaths, new Function<String, String>() { |
|
127 |
@Override |
|
128 |
public String apply(final String path) { |
|
129 |
return path + SEMICOLON + promoteJobMapperClass; |
|
130 |
} |
|
131 |
}))); |
|
132 |
|
|
133 |
params.put("hbase.mapred.outputtable", dataTable); |
|
134 |
params.put("hbase.mapreduce.outputtable", dataTable); |
|
135 |
|
|
136 |
log.info("promoting HDFS rawsets: " + existingPaths); |
|
137 |
|
|
138 |
executeHDFS("promoteMultipleActionSetsJob", params, callback); |
|
139 |
} |
|
140 |
|
|
141 |
private void executeHDFS(final String jobName, final Map<String, String> params, final JobCallback callback) throws ActionManagerException { |
|
142 |
|
|
143 |
if (params == null || params.isEmpty()) { throw new ActionManagerException("Missing HDFS paths"); } |
|
144 |
|
|
145 |
executor.execute(new Runnable() { |
|
146 |
|
|
147 |
@Override |
|
148 |
public void run() { |
|
149 |
try { |
|
150 |
final String serviceId = findHadoopServiceProfileID(); |
|
151 |
final BlackboardJob bbJob = blackboardClientHandler.newJob(serviceId); |
|
152 |
|
|
153 |
bbJob.setAction(HadoopBlackboardActions.SUBMIT_MAPREDUCE_JOB.toString()); |
|
154 |
bbJob.getParameters().put("job.name", jobName); |
|
155 |
bbJob.getParameters().put("cluster", ClusterName.DM.toString()); |
|
156 |
bbJob.getParameters().putAll(params); |
|
157 |
|
|
158 |
jobRegistry.registerJobListener(bbJob, new BlackboardJobListener(callback)); |
|
159 |
blackboardClientHandler.assign(bbJob); |
|
160 |
} catch (Exception e) { |
|
161 |
log.error("Error launching m/r job: " + jobName, e); |
|
162 |
throw new RuntimeException("Error launching m/r job: " + jobName, e); |
|
163 |
} |
|
164 |
} |
|
165 |
|
|
166 |
}); |
|
167 |
} |
|
168 |
|
|
169 |
public void executeGarbageActions() throws ISRegistryException, HadoopServiceException, ParseException, ActionManagerException { |
|
170 |
final String basePath = isClient.getBasePathHDFS(); |
|
171 |
final Integer garbageRetain = Integer.valueOf(isClient.getGarbageRetainThreshold()); |
|
172 |
final Long dateLimit = DateUtils.now() - Long.valueOf(isClient.getGarbageTimeMargin()); |
|
173 |
|
|
174 |
final HadoopService hadoopService = serviceLocator.getService(HadoopService.class); |
|
175 |
final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class); |
|
176 |
|
|
177 |
for (ActionManagerSet set : isClient.listValidSets()) { |
|
178 |
for (RawSet expired : Iterables.limit(set.getExpired(), garbageRetain)) { |
|
179 |
final Date d = org.apache.commons.lang.time.DateUtils.parseDate(expired.getLastUpdate(), ActionManagerSet.DATE_PATTERNS); |
|
180 |
if (d.getTime() < dateLimit) { |
|
181 |
|
|
182 |
log.info(String.format("removing raw action set %s/%s", set.getId(), expired.getId())); |
|
183 |
hadoopService.deleteHdfsPath(ClusterName.DM.toString(), basePath + "/" + set.getDirectory() + "/" + expired.getId()); |
|
184 |
|
|
185 |
final String xUpdate = String.format(UPDATE_ACTION_PROFILE, set.getId(), expired.getId()); |
|
186 |
log.info(String.format("updating ActionSet profile: %s", xUpdate)); |
|
187 |
isRegistry.executeXUpdate(xUpdate); |
|
188 |
} |
|
189 |
} |
|
190 |
} |
|
191 |
} |
|
192 |
|
|
193 |
private String findHadoopServiceProfileID() throws Exception { |
|
194 |
return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery( |
|
195 |
"collection('/db/DRIVER/ServiceResources/HadoopServiceResourceType')//RESOURCE_IDENTIFIER/@value/string()"); |
|
196 |
} |
|
197 |
|
|
198 |
public String getActionTable() { |
|
199 |
return actionTable; |
|
200 |
} |
|
201 |
|
|
202 |
@Required |
|
203 |
public void setActionTable(final String actionTable) { |
|
204 |
this.actionTable = actionTable; |
|
205 |
} |
|
206 |
|
|
207 |
public String getDataTable() { |
|
208 |
return dataTable; |
|
209 |
} |
|
210 |
|
|
211 |
@Required |
|
212 |
public void setDataTable(final String dataTable) { |
|
213 |
this.dataTable = dataTable; |
|
214 |
} |
|
215 |
|
|
216 |
public ISClient getInfomationServiceClient() { |
|
217 |
return infomationServiceClient; |
|
218 |
} |
|
219 |
|
|
220 |
@Required |
|
221 |
public void setInfomationServiceClient(final ISClient infomationServiceClient) { |
|
222 |
this.infomationServiceClient = infomationServiceClient; |
|
223 |
} |
|
224 |
|
|
225 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/inspector/ActionManagerController.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.inspector; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Set; |
|
5 |
|
|
6 |
import eu.dnetlib.actionmanager.ActionManagerConstants.ACTION_TYPE; |
|
7 |
import eu.dnetlib.actionmanager.blackboard.JobLauncher; |
|
8 |
import eu.dnetlib.actionmanager.common.Provenance; |
|
9 |
import eu.dnetlib.actionmanager.is.ISClient; |
|
10 |
import eu.dnetlib.actionmanager.rmi.ActionManagerException; |
|
11 |
import eu.dnetlib.actionmanager.set.ActionManagerSet; |
|
12 |
import eu.dnetlib.enabling.inspector.AbstractInspectorController; |
|
13 |
import org.apache.commons.lang.StringUtils; |
|
14 |
import org.springframework.beans.factory.annotation.Autowired; |
|
15 |
import org.springframework.stereotype.Controller; |
|
16 |
import org.springframework.ui.ModelMap; |
|
17 |
import org.springframework.web.bind.annotation.ModelAttribute; |
|
18 |
import org.springframework.web.bind.annotation.RequestMapping; |
|
19 |
import org.springframework.web.bind.annotation.RequestParam; |
|
20 |
|
|
21 |
@Controller |
|
22 |
public class ActionManagerController extends AbstractInspectorController { |
|
23 |
|
|
24 |
@Autowired |
|
25 |
private ISClient isClient; |
|
26 |
|
|
27 |
@Autowired |
|
28 |
private JobLauncher jobLauncher; |
|
29 |
|
|
30 |
@ModelAttribute(value = "types") |
|
31 |
public ACTION_TYPE[] listActionTypes() { |
|
32 |
return ACTION_TYPE.values(); |
|
33 |
} |
|
34 |
|
|
35 |
@ModelAttribute(value = "provenances") |
|
36 |
public Provenance[] listProvenances() { |
|
37 |
return Provenance.values(); |
|
38 |
} |
|
39 |
|
|
40 |
@ModelAttribute(value = "sets") |
|
41 |
public List<ActionManagerSet> listSets() throws ActionManagerException { |
|
42 |
return isClient.listSets(); |
|
43 |
} |
|
44 |
|
|
45 |
@RequestMapping("/inspector/actions/createSet.do") |
|
46 |
public String createSet(final ModelMap map, |
|
47 |
@RequestParam(value = "id", required = true) final String id, |
|
48 |
@RequestParam(value = "name", required = true) final String name, |
|
49 |
@RequestParam(value = "directory", required = true) final String directory) throws Exception { |
|
50 |
isClient.registerSetProfile(new ActionManagerSet(id, name, directory)); |
|
51 |
return "redirect:listSets.do"; |
|
52 |
} |
|
53 |
|
|
54 |
@RequestMapping("/inspector/actions/listSets.do") |
|
55 |
public void createSet(final ModelMap map) throws Exception { |
|
56 |
map.addAttribute("sets", isClient.listValidSets()); |
|
57 |
} |
|
58 |
|
|
59 |
@RequestMapping("/inspector/actions/commit.do") |
|
60 |
public void commit(final ModelMap map, @RequestParam(value = "sets", required = false) final Set<String> sets) throws Exception { |
|
61 |
jobLauncher.executePromoteFromHDFSJob(sets, null); |
|
62 |
map.addAttribute("message", "Job launched !!!"); |
|
63 |
} |
|
64 |
|
|
65 |
@RequestMapping("/inspector/actions/garbage.do") |
|
66 |
public void garbage(final ModelMap map) throws Exception { |
|
67 |
jobLauncher.executeGarbageActions(); |
|
68 |
map.addAttribute("message", "Job launched !!!"); |
|
69 |
} |
|
70 |
|
|
71 |
@RequestMapping("/inspector/actions/deleteSet.do") |
|
72 |
public String deleteSet(final ModelMap map, @RequestParam(value = "set", required = true) final String set) throws Exception { |
|
73 |
map.addAttribute("allSets", JobLauncher.ALL_SETS); |
|
74 |
if (set != null && !set.isEmpty()) { |
|
75 |
isClient.deleteSetProfile(set); |
|
76 |
} |
|
77 |
return "redirect:listSets.do"; |
|
78 |
} |
|
79 |
|
|
80 |
@RequestMapping("/inspector/actions/linkActionSetProfile.do") |
|
81 |
public String linkProfile(@RequestParam(value = "set", required = true) final String set) throws Exception { |
|
82 |
if (StringUtils.isBlank(set)) { |
|
83 |
throw new IllegalArgumentException("missing set id"); |
|
84 |
} |
|
85 |
final String profileId = isClient.getProfileId(set); |
|
86 |
return "redirect:/mvc/inspector/index.do/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType/" + profileId + "/show"; |
|
87 |
} |
|
88 |
|
|
89 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/ActionManagerServiceImpl.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import javax.jws.WebParam; |
|
5 |
|
|
6 |
import eu.dnetlib.actionmanager.is.ISClient; |
|
7 |
import eu.dnetlib.actionmanager.rmi.ActionManagerException; |
|
8 |
import eu.dnetlib.actionmanager.rmi.ActionManagerService; |
|
9 |
import eu.dnetlib.actionmanager.set.ActionManagerSet; |
|
10 |
import eu.dnetlib.data.hadoop.config.ClusterName; |
|
11 |
import eu.dnetlib.data.hadoop.rmi.HadoopService; |
|
12 |
import eu.dnetlib.data.hadoop.rmi.HadoopServiceException; |
|
13 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
14 |
import eu.dnetlib.enabling.tools.AbstractBaseService; |
|
15 |
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler; |
|
16 |
import org.apache.hadoop.fs.Path; |
|
17 |
import org.springframework.beans.factory.annotation.Autowired; |
|
18 |
import org.springframework.beans.factory.annotation.Required; |
|
19 |
|
|
20 |
public class ActionManagerServiceImpl extends AbstractBaseService implements ActionManagerService { |
|
21 |
|
|
22 |
private NotificationHandler notificationHandler; |
|
23 |
|
|
24 |
@Autowired |
|
25 |
private ISClient isClient; |
|
26 |
|
|
27 |
@Autowired |
|
28 |
private UniqueServiceLocator serviceLocator; |
|
29 |
|
|
30 |
@Override |
|
31 |
public String createSet(@WebParam(name = "set") final ActionManagerSet set) throws ActionManagerException { |
|
32 |
try { |
|
33 |
final String basePath = isClient.getBasePathHDFS(); |
|
34 |
final Path actionSetPath = new Path(basePath + "/" + set.getId()); |
|
35 |
serviceLocator.getService(HadoopService.class).createHdfsDirectory(ClusterName.DM.toString(), actionSetPath.toString(), true); |
|
36 |
return isClient.registerSetProfile(set); |
|
37 |
} catch (HadoopServiceException e) { |
|
38 |
throw new ActionManagerException(e); |
|
39 |
} |
|
40 |
} |
|
41 |
|
|
42 |
@Override |
|
43 |
public boolean deleteSet(@WebParam(name = "set") final String setId) throws ActionManagerException { |
|
44 |
try { |
|
45 |
final String basePath = isClient.getBasePathHDFS(); |
|
46 |
final Path actionSetPath = new Path(basePath + "/" + setId); |
|
47 |
serviceLocator.getService(HadoopService.class).deleteHdfsPath(ClusterName.DM.toString(), actionSetPath.toString()); |
|
48 |
return isClient.deleteSetProfile(setId); |
|
49 |
} catch (HadoopServiceException e) { |
|
50 |
throw new ActionManagerException(e); |
|
51 |
} |
|
52 |
} |
|
53 |
|
|
54 |
@Override |
|
55 |
public List<ActionManagerSet> ListSets() throws ActionManagerException { |
|
56 |
return isClient.listValidSets(); |
|
57 |
} |
|
58 |
|
|
59 |
@Override |
|
60 |
public void notify(final String subscriptionId, final String topic, final String isId, final String message) { |
|
61 |
notificationHandler.notified(subscriptionId, topic, isId, message); |
|
62 |
} |
|
63 |
|
|
64 |
public NotificationHandler getNotificationHandler() { |
|
65 |
return notificationHandler; |
|
66 |
} |
|
67 |
|
|
68 |
@Required |
|
69 |
public void setNotificationHandler(final NotificationHandler notificationHandler) { |
|
70 |
this.notificationHandler = notificationHandler; |
|
71 |
} |
|
72 |
|
|
73 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/is/ISClient.java | ||
---|---|---|
1 |
package eu.dnetlib.actionmanager.is; |
|
2 |
|
|
3 |
import java.io.StringReader; |
|
4 |
import java.util.List; |
|
5 |
import java.util.NoSuchElementException; |
|
6 |
import java.util.Set; |
|
7 |
import javax.annotation.Resource; |
|
8 |
import javax.xml.ws.Endpoint; |
|
9 |
|
|
10 |
import com.google.common.collect.Iterables; |
|
11 |
import com.google.common.collect.Lists; |
|
12 |
import com.google.common.collect.Sets; |
|
13 |
import eu.dnetlib.actionmanager.rmi.ActionManagerException; |
|
14 |
import eu.dnetlib.actionmanager.set.ActionManagerSet; |
|
15 |
import eu.dnetlib.actionmanager.set.ActionManagerSet.ImpactTypes; |
|
16 |
import eu.dnetlib.actionmanager.set.RawSet; |
|
17 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
|
18 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
19 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; |
|
20 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
21 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
22 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
23 |
import eu.dnetlib.soap.EndpointReferenceBuilder; |
|
24 |
import org.antlr.stringtemplate.StringTemplate; |
|
25 |
import org.apache.commons.logging.Log; |
|
26 |
import org.apache.commons.logging.LogFactory; |
|
27 |
import org.dom4j.Attribute; |
|
28 |
import org.dom4j.Document; |
|
29 |
import org.dom4j.Element; |
|
30 |
import org.dom4j.io.SAXReader; |
|
31 |
import org.springframework.beans.factory.annotation.Required; |
|
32 |
|
|
33 |
public class ISClient { |
|
34 |
|
|
35 |
private static final Log log = LogFactory.getLog(ISClient.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
36 |
@Resource |
|
37 |
private UniqueServiceLocator serviceLocator; |
|
38 |
/** |
|
39 |
* Template for ActionManagerSet profiles |
|
40 |
*/ |
|
41 |
private StringTemplate actionManagerSetDsTemplate; |
|
42 |
/** |
|
43 |
* service endpoint. |
|
44 |
*/ |
|
45 |
private Endpoint endpoint; |
|
46 |
/** |
|
47 |
* endpoint builder. |
|
48 |
*/ |
|
49 |
private EndpointReferenceBuilder<Endpoint> eprBuilder; |
|
50 |
|
|
51 |
public String registerSetProfile(final ActionManagerSet set) throws ActionManagerException { |
|
52 |
if (existsSet(set.getId())) { throw new ActionManagerException("Set " + set.getId() + " already registered"); } |
|
53 |
try { |
|
54 |
StringTemplate template = new StringTemplate(actionManagerSetDsTemplate.getTemplate()); |
|
55 |
template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint)); |
|
56 |
template.setAttribute("set", set); |
|
57 |
template.setAttribute("latest", RawSet.newInstance()); |
|
58 |
return serviceLocator.getService(ISRegistryService.class).registerProfile(template.toString()); |
|
59 |
} catch (ISRegistryException e) { |
|
60 |
throw new ActionManagerException("Error registering set " + set, e); |
|
61 |
} |
|
62 |
} |
|
63 |
|
|
64 |
public List<ActionManagerSet> listSets() throws ActionManagerException { |
|
65 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') return $x"; |
|
66 |
return getActionManagerSets(q); |
|
67 |
} |
|
68 |
|
|
69 |
public List<ActionManagerSet> listValidSets() throws ActionManagerException { |
|
70 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') \n" |
|
71 |
+ "where string-length($x//RAW_SETS/LATEST/@id/string()) > 0\n" |
|
72 |
+ "return $x"; |
|
73 |
|
|
74 |
return getActionManagerSets(q); |
|
75 |
} |
|
76 |
|
|
77 |
private List<ActionManagerSet> getActionManagerSets(final String q) throws ActionManagerException { |
|
78 |
final List<ActionManagerSet> list = Lists.newArrayList(); |
|
79 |
|
|
80 |
try { |
|
81 |
final String basePath = getBasePathHDFS(); |
|
82 |
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q)) { |
|
83 |
list.add(getActionManagerSet(basePath, s)); |
|
84 |
} |
|
85 |
} catch (ISLookUpException e) { |
|
86 |
log.error("Error accessing Sets, using query: " + q); |
|
87 |
} |
|
88 |
return list; |
|
89 |
} |
|
90 |
|
|
91 |
public ActionManagerSet getSet(final String setId) throws ActionManagerException, ISLookUpException { |
|
92 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') " |
|
93 |
+ "where $x//SET/@id = '" + setId + "' return $x"; |
|
94 |
|
|
95 |
try { |
|
96 |
final String basePath = getBasePathHDFS(); |
|
97 |
final String setProfile = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q); |
|
98 |
return getActionManagerSet(basePath, setProfile); |
|
99 |
} catch (ISLookUpException e) { |
|
100 |
throw new ActionManagerException("Error accessing Sets, using query: " + q); |
|
101 |
} |
|
102 |
} |
|
103 |
|
|
104 |
private ActionManagerSet getActionManagerSet(final String basePath, final String profile) throws ActionManagerException { |
|
105 |
final SAXReader reader = new SAXReader(); |
|
106 |
final ActionManagerSet set = new ActionManagerSet(); |
|
107 |
|
|
108 |
try { |
|
109 |
final Document doc = reader.read(new StringReader(profile)); |
|
110 |
|
|
111 |
set.setId(doc.valueOf("//SET/@id").trim()); |
|
112 |
set.setName(doc.valueOf("//SET").trim()); |
|
113 |
set.setImpact(ImpactTypes.valueOf(doc.valueOf("//IMPACT").trim())); |
|
114 |
set.setLatest(doc.valueOf("//RAW_SETS/LATEST/@id"), doc.valueOf("//RAW_SETS/LATEST/@creationDate"), doc.valueOf("//RAW_SETS/LATEST/@lastUpdate")); |
|
115 |
set.setDirectory(doc.valueOf("//SET/@directory")); |
|
116 |
final List expiredNodes = doc.selectNodes("//RAW_SETS/EXPIRED"); |
|
117 |
if (expiredNodes != null) { |
|
118 |
for (int i = 0; i < expiredNodes.size(); i++) { |
|
119 |
Element ex = (Element) expiredNodes.get(i); |
|
120 |
set.addExpired(ex.attributeValue("id"), ex.attributeValue("creationDate"), ex.attributeValue("lastUpdate")); |
|
121 |
} |
|
122 |
} |
|
123 |
|
|
124 |
final StringBuilder sb = new StringBuilder(); |
|
125 |
sb.append(basePath); |
|
126 |
sb.append("/"); |
|
127 |
sb.append(doc.valueOf("//SET/@directory")); |
|
128 |
sb.append("/"); |
|
129 |
sb.append(doc.valueOf("//RAW_SETS/LATEST/@id")); |
|
130 |
set.setPathToLatest(sb.toString()); |
|
131 |
|
|
132 |
return set; |
|
133 |
} catch (Exception e) { |
|
134 |
throw new ActionManagerException("Error creating set from profile: " + profile, e); |
|
135 |
} |
|
136 |
} |
|
137 |
|
|
138 |
public boolean deleteSetProfile(final String setId) throws ActionManagerException { |
|
139 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + setId |
|
140 |
+ "' return $x//RESOURCE_IDENTIFIER/@value/string()"; |
|
141 |
|
|
142 |
try { |
|
143 |
final String profId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q); |
|
144 |
return serviceLocator.getService(ISRegistryService.class).deleteProfile(profId); |
|
145 |
} catch (Exception e) { |
|
146 |
log.error("Error deleting set " + setId, e); |
|
147 |
throw new ActionManagerException("Error deleting set " + setId, e); |
|
148 |
} |
|
149 |
} |
|
150 |
|
|
151 |
public boolean existsSet(final String id) throws ActionManagerException { |
|
152 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + id |
|
153 |
+ "' return $x"; |
|
154 |
try { |
|
155 |
return !serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q).isEmpty(); |
|
156 |
} catch (ISLookUpException e) { |
|
157 |
log.error("Error accessing Sets, using query: " + q, e); |
|
158 |
throw new ActionManagerException("Error running xquery: " + q, e); |
|
159 |
} |
|
160 |
} |
|
161 |
|
|
162 |
public boolean existsRawSet(final String id) throws ActionManagerException { |
|
163 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//RAW_SETS/*/@id = '" + id |
|
164 |
+ "' return $x"; |
|
165 |
try { |
|
166 |
return !serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q).isEmpty(); |
|
167 |
} catch (ISLookUpException e) { |
|
168 |
log.error("Error accessing RawSets, using query: " + q, e); |
|
169 |
throw new ActionManagerException("Error running xquery: " + q, e); |
|
170 |
} |
|
171 |
} |
|
172 |
|
|
173 |
public Set<String> listValidRawSets() throws ActionManagerException { |
|
174 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') return $x//RAW_SETS/LATEST/@id/string()"; |
|
175 |
try { |
|
176 |
final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q); |
|
177 |
return Sets.newHashSet(list); |
|
178 |
} catch (Exception e) { |
|
179 |
log.error("Error running xquery: " + q, e); |
|
180 |
throw new ActionManagerException("Error running xquery: " + q, e); |
|
181 |
} |
|
182 |
} |
|
183 |
|
|
184 |
public RawSet geLatestRawSet(final String set) throws ActionManagerException { |
|
185 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set |
|
186 |
+ "' return concat(' ' , $x//RAW_SETS/LATEST/@id, ' @@@ ', $x//RAW_SETS/LATEST/@creationDate, ' @@@ ', $x//RAW_SETS/LATEST/@lastUpdate, ' ')"; |
|
187 |
try { |
|
188 |
String[] arr = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q).split("@@@"); |
|
189 |
return RawSet.newInstance(arr[0].trim(), arr[1].trim(), arr[2].trim()); |
|
190 |
} catch (ISLookUpException e) { |
|
191 |
log.error("Error accessing Sets, using query: " + q, e); |
|
192 |
throw new ActionManagerException("Error running xquery: " + q, e); |
|
193 |
} |
|
194 |
} |
|
195 |
|
|
196 |
public String getGarbageTimeMargin() throws ActionManagerException { |
|
197 |
return queryServiceProperty("garbageTimeMargin"); |
|
198 |
} |
|
199 |
|
|
200 |
public String getBasePathHDFS() throws ActionManagerException { |
|
201 |
return queryServiceProperty("basePath"); |
|
202 |
} |
|
203 |
|
|
204 |
public String getGarbageRetainThreshold() throws ActionManagerException { |
|
205 |
return queryServiceProperty("garbageRetainThreshold"); |
|
206 |
} |
|
207 |
|
|
208 |
private String queryServiceProperty(final String propertyName) throws ActionManagerException { |
|
209 |
final String q = "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='" |
|
210 |
+ propertyName + "']/@value/string()"; |
|
211 |
log.debug("quering for service property: " + q); |
|
212 |
try { |
|
213 |
final List<String> value = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q); |
|
214 |
return Iterables.getOnlyElement(value); |
|
215 |
} catch (ISLookUpException e) { |
|
216 |
String msg = "Error accessing service profile, using query: " + q; |
|
217 |
log.error(msg, e); |
|
218 |
throw new ActionManagerException(msg, e); |
|
219 |
} catch (NoSuchElementException e) { |
|
220 |
String msg = "missing service property: " + propertyName; |
|
221 |
log.error(msg, e); |
|
222 |
throw new ActionManagerException(msg, e); |
|
223 |
} catch (IllegalArgumentException e) { |
|
224 |
String msg = "found more than one service property: " + propertyName; |
|
225 |
log.error(msg, e); |
|
226 |
throw new ActionManagerException(msg, e); |
|
227 |
} |
|
228 |
} |
|
229 |
|
|
230 |
public void addLatestRawSet(final String set, final RawSet rawSet) throws ActionManagerException { |
|
231 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set |
|
232 |
+ "' return $x"; |
|
233 |
try { |
|
234 |
final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q); |
|
235 |
final Document doc = new SAXReader().read(new StringReader(profile)); |
|
236 |
final String profId = doc.valueOf("//RESOURCE_IDENTIFIER/@value"); |
|
237 |
final Element latest = (Element) doc.selectSingleNode("//RAW_SETS/LATEST"); |
|
238 |
final Element expired = ((Element) doc.selectSingleNode("//RAW_SETS")).addElement("EXPIRED"); |
|
239 |
|
|
240 |
for (Object o : latest.attributes()) { |
|
241 |
final Attribute a = (Attribute) o; |
|
242 |
expired.addAttribute(a.getName(), a.getValue()); |
|
243 |
} |
|
244 |
|
|
245 |
latest.addAttribute("id", rawSet.getId()); |
|
246 |
latest.addAttribute("creationDate", rawSet.getCreationDate()); |
|
247 |
latest.addAttribute("lastUpdate", rawSet.getLastUpdate()); |
|
248 |
|
|
249 |
serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ActionManagerSetDSResourceType"); |
|
250 |
} catch (Exception e) { |
|
251 |
log.error("Error updating profile of set: " + set); |
|
252 |
throw new ActionManagerException("Error running xquery: " + q, e); |
|
253 |
} |
|
254 |
} |
|
255 |
|
|
256 |
public void updateLastUpdate(final String rawSet) throws ActionManagerException { |
|
257 |
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType')//RAW_SETS/*[@id='" + rawSet |
|
258 |
+ "']/@lastUpdate return update replace $x with '" + DateUtils.now_ISO8601() + "'"; |
|
259 |
|
|
260 |
try { |
|
261 |
serviceLocator.getService(ISRegistryService.class).executeXUpdate(q); |
|
262 |
} catch (Exception e) { |
|
263 |
log.error("Error updating lastUpdate using query: " + q, e); |
|
264 |
throw new ActionManagerException("Error updating lastUpdate using query: " + q, e); |
|
265 |
} |
|
266 |
} |
|
267 |
|
|
268 |
public StringTemplate getActionManagerSetDsTemplate() { |
|
269 |
return actionManagerSetDsTemplate; |
|
270 |
} |
|
271 |
|
|
272 |
public String getProfileId(final String set) throws ISLookUpException { |
|
273 |
final String xquery = "substring-before(for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') " |
|
274 |
+ "return $x[.//SET/@id = '%s']//RESOURCE_IDENTIFIER/@value/string(), '_')"; |
|
275 |
return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(String.format(xquery, set)); |
|
276 |
} |
|
277 |
|
|
278 |
@Required |
|
279 |
public void setActionManagerSetDsTemplate(final StringTemplate actionManagerSetDsTemplate) { |
|
280 |
this.actionManagerSetDsTemplate = actionManagerSetDsTemplate; |
|
281 |
} |
|
282 |
|
|
283 |
public Endpoint getEndpoint() { |
|
284 |
return endpoint; |
|
285 |
} |
|
286 |
|
|
287 |
@Required |
|
288 |
public void setEndpoint(final Endpoint endpoint) { |
|
289 |
this.endpoint = endpoint; |
|
290 |
} |
|
291 |
|
|
292 |
public EndpointReferenceBuilder<Endpoint> getEprBuilder() { |
|
293 |
return eprBuilder; |
|
294 |
} |
|
295 |
|
|
296 |
@Required |
|
297 |
public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) { |
|
298 |
this.eprBuilder = eprBuilder; |
|
299 |
} |
|
300 |
|
|
301 |
} |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/listSets.st | ||
---|---|---|
1 |
$inspector/actions/actionMaster(ait={ |
|
2 |
|
|
3 |
<style type="text/css"> |
|
4 |
#setList { margin-top: 12px; } |
|
5 |
#setList td { padding: 4px; padding-left: 8px; padding-right: 8px; font-size: 11px;} |
|
6 |
#setList th { font-weight: bold; border-bottom: 1px solid #999; font-size: 14px; } |
|
7 |
</style> |
|
8 |
|
|
9 |
<br /><br /> |
|
10 |
<form method="get" action="createSet.do"> |
|
11 |
<b>New set:</b> |
|
12 |
ID: <input type="text" name="id" /> |
|
13 |
Name: <input type="text" name="name" /> |
|
14 |
Directory: <input type="text" name="directory" /> |
|
15 |
<input type="submit" value="create"> |
|
16 |
</form> |
|
17 |
<br /> |
|
18 |
<table id="setList" cellspacing="0px"> |
|
19 |
<thead> |
|
20 |
<tr> |
|
21 |
<th style="width: 100px">id</th> |
|
22 |
<th style="width: 240px">name</th> |
|
23 |
<th style="width: 120px">impact</th> |
|
24 |
<th style="width: 450px">latest raw set</th> |
|
25 |
<th style="width: 200px">latest raw set creation</th> |
|
26 |
<th style="width: 200px">latest raw set update</th> |
|
27 |
<th style="width: 100px"> </th> |
|
28 |
</tr> |
|
29 |
</thead> |
|
30 |
<tbody> |
|
31 |
$sets:{s| |
|
32 |
<tr> |
|
33 |
<td><a href="linkActionSetProfile.do?set=$s.id$">$s.id$</a></td> |
|
34 |
<td>$s.name$</td> |
|
35 |
<td>$s.impact$</td> |
|
36 |
<td>$s.latest.id$</td> |
|
37 |
<td>$s.latest.creationDate$</td> |
|
38 |
<td>$s.latest.lastUpdate$</td> |
|
39 |
<td><a href="deleteSet.do?set=$s.id$">[delete]</a></td> |
|
40 |
</tr> |
|
41 |
}$ |
|
42 |
</tbody> |
|
43 |
</table> |
|
44 |
|
|
45 |
})$ |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/listAll.st | ||
---|---|---|
1 |
$inspector/actions/actionMaster(ait={ |
|
2 |
<p> |
|
3 |
<b>All actions:</b> |
|
4 |
</p> |
|
5 |
|
|
6 |
<table id="actionList" cellspacing="0px"> |
|
7 |
<tr class="header"> |
|
8 |
<td>action ID</td> |
|
9 |
<td>set</td> |
|
10 |
<td>date</td> |
|
11 |
</tr> |
|
12 |
$actions:{a| |
|
13 |
<tr> |
|
14 |
<td><b>$a.rowId$</b></td> |
|
15 |
<td>$a.set$</td> |
|
16 |
<td>$a.date$</td> |
|
17 |
</tr> |
|
18 |
}$ |
|
19 |
</table> |
|
20 |
<br /><br /> |
|
21 |
$if(next)$<a href="?type=$type$&start=$next$">[Continue >>]</a>$endif$ |
|
22 |
|
|
23 |
})$ |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/commit.st | ||
---|---|---|
1 |
$inspector/actions/actionMaster(ait={ |
|
2 |
<br /> |
|
3 |
<form method="get"> |
|
4 |
<b>Sets (No selection for ALL SETS):</b><br /><br /> |
|
5 |
<select name="sets" multiple="multiple" size="10"> |
|
6 |
$sets:{s|<option value="$s.id$">$s.name$</option>}$ |
|
7 |
</select><br /><br /> |
|
8 |
<input type="submit" value="promote"> |
|
9 |
</form> |
|
10 |
<br /> |
|
11 |
<hr /> |
|
12 |
<br /> |
|
13 |
$if(message)$ |
|
14 |
<p> |
|
15 |
<b>$message$</b><br /><br /> |
|
16 |
<a href="../msroAllProcesses.do">See orchestration processes</a><br /><br /> |
|
17 |
</p> |
|
18 |
$endif$ |
|
19 |
})$ |
|
20 |
|
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/garbage.st | ||
---|---|---|
1 |
$inspector/actions/actionMaster(ait={ |
|
2 |
<br /><br /> |
|
3 |
$if(message)$ |
|
4 |
<p> |
|
5 |
<b>$message$</b><br /><br /> |
|
6 |
<a href="../msroAllProcesses.do">See orchestration processes</a><br /><br /> |
|
7 |
</p> |
|
8 |
$else$ |
|
9 |
|
|
10 |
})$ |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/actionMaster.st | ||
---|---|---|
1 |
$inspector/master(it={ |
|
2 |
<style type="text/css"> |
|
3 |
#actionList { margin-top: 12px; } |
|
4 |
#actionList td { padding: 4px; padding-left: 8px; padding-right: 8px; font-size: 11px;} |
|
5 |
#actionList tr.header td { font-weight: bold; border-bottom: 1px solid #999; font-size: 14px; } |
|
6 |
</style> |
|
7 |
|
|
8 |
<h2>Action Manager service</h2> |
|
9 |
|
|
10 |
<p><b>Operation:</b> |
|
11 |
<a href="commit.do">promote actions</a> | |
|
12 |
<a href="garbage.do">garbage actions</a> | |
|
13 |
<a href="listSets.do">sets</a> |
|
14 |
</p> |
|
15 |
<hr /> |
|
16 |
|
|
17 |
$ait$ |
|
18 |
|
|
19 |
})$ |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/profiles/promoteActions.xml | ||
---|---|---|
1 |
<RESOURCE_PROFILE> |
|
2 |
<HEADER> |
|
3 |
<RESOURCE_IDENTIFIER |
|
4 |
value="8bb6c559-edf3-4da1-87d7-cdee4fba21dd_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ==" /> |
|
5 |
<RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType" /> |
|
6 |
<RESOURCE_KIND value="HadoopJobConfigurationDSResources" /> |
|
7 |
<RESOURCE_URI value="" /> |
|
8 |
<DATE_OF_CREATION value="2001-12-31T12:00:00" /> |
|
9 |
</HEADER> |
|
10 |
<BODY> |
|
11 |
<HADOOP_JOB name="promoteActionsJob" type="mapreduce"> |
|
12 |
<DESCRIPTION>map only job that promote actions from a set to the data table</DESCRIPTION> |
|
13 |
<STATIC_CONFIGURATION> |
|
14 |
<PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.actions2.PromoteActionsMapper" /> |
|
15 |
<PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat" /> |
|
16 |
<PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableOutputFormat" /> |
|
17 |
<PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text" /> |
|
18 |
<PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.hbase.client.Put" /> |
|
19 |
<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" /> |
|
20 |
<PROPERTY key="mapreduce.map.speculative" value="false" /> |
|
21 |
<PROPERTY key="mapred.reduce.tasks" value="0" /> |
|
22 |
<!-- <PROPERTY key="job.lib" value="/tmp/dnet-mapreduce-jobs-0.0.3-SNAPSHOT-jar-with-dependencies.jar" /> --> |
|
23 |
</STATIC_CONFIGURATION> |
|
24 |
|
|
25 |
<JOB_INTERFACE> |
|
26 |
<PARAM name="hbase.mapred.inputtable" required="true" description="source hbase table" /> |
|
27 |
<PARAM name="hbase.mapred.outputtable" required="true" description="target hbase table" /> |
|
28 |
<PARAM name="hbase.mapreduce.inputtable" required="true" description="source hbase table" /> |
|
29 |
<PARAM name="hbase.mapreduce.outputtable" required="true" description="target hbase table" /> |
|
30 |
<PARAM name="latestRawSets" required="true" description="the latest action raw sets" /> |
|
31 |
</JOB_INTERFACE> |
|
32 |
|
|
33 |
<SCAN> |
|
34 |
<FILTERS operator="MUST_PASS_ALL"> |
|
35 |
<FILTER type="prefix" value="aac|" /> |
|
36 |
</FILTERS> |
|
37 |
<FAMILIES /> |
|
38 |
</SCAN> |
|
39 |
|
|
40 |
</HADOOP_JOB> |
|
41 |
|
|
42 |
<STATUS> |
|
43 |
<LAST_SUBMISSION_DATE value="2001-12-31T12:00:00" /> |
|
44 |
<RUNNING_INSTANCES value="0" /> |
|
45 |
<CUMULATIVE_RUN value="0" /> |
|
46 |
</STATUS> |
|
47 |
|
|
48 |
<SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS> |
|
49 |
</BODY> |
|
50 |
</RESOURCE_PROFILE> |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/profiles/garbageActions.xml | ||
---|---|---|
1 |
<RESOURCE_PROFILE> |
|
2 |
<HEADER> |
|
3 |
<RESOURCE_IDENTIFIER |
|
4 |
value="58c652e5-4789-47fe-87f2-c511ba6f72ea_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ==" /> |
|
5 |
<RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType" /> |
|
6 |
<RESOURCE_KIND value="HadoopJobConfigurationDSResources" /> |
|
7 |
<RESOURCE_URI value="" /> |
|
8 |
<DATE_OF_CREATION value="2001-12-31T12:00:00" /> |
|
9 |
</HEADER> |
|
10 |
<BODY> |
|
11 |
<HADOOP_JOB name="garbageActionsJob" type="mapreduce"> |
|
12 |
<DESCRIPTION>map only job that delete actions from a set |
|
13 |
</DESCRIPTION> |
|
14 |
<STATIC_CONFIGURATION> |
|
15 |
<PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.actions2.GarbageActionsMapper" /> |
|
16 |
<PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat" /> |
|
17 |
<PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableOutputFormat" /> |
|
18 |
<PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text" /> |
|
19 |
<PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.hbase.client.Delete" /> |
|
20 |
<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" /> |
|
21 |
<PROPERTY key="mapreduce.map.speculative" value="false" /> |
|
22 |
<PROPERTY key="mapred.reduce.tasks" value="0" /> |
|
23 |
<!-- <PROPERTY key="job.lib" value="/tmp/dnet-mapreduce-jobs-0.0.3-SNAPSHOT-jar-with-dependencies.jar" /> --> |
|
24 |
</STATIC_CONFIGURATION> |
|
25 |
<JOB_INTERFACE> |
|
26 |
<PARAM name="hbase.mapred.inputtable" required="true" description="source hbase table" /> |
|
27 |
<PARAM name="hbase.mapred.outputtable" required="true" description="target hbase table" /> |
|
28 |
<PARAM name="hbase.mapreduce.inputtable" required="true" description="source hbase table" /> |
|
29 |
<PARAM name="hbase.mapreduce.outputtable" required="true" description="target hbase table" /> |
|
30 |
<PARAM name="latestRawSets" required="true" description="the latest action raw sets" /> |
|
31 |
</JOB_INTERFACE> |
|
32 |
<SCAN> |
|
33 |
<FILTERS/> |
|
34 |
<FAMILIES> |
|
35 |
<FAMILY value="set" /> |
|
36 |
</FAMILIES> |
|
37 |
</SCAN> |
|
38 |
</HADOOP_JOB> |
|
39 |
<STATUS> |
|
40 |
<LAST_SUBMISSION_DATE value="2001-12-31T12:00:00" /> |
|
41 |
<RUNNING_INSTANCES value="0" /> |
|
42 |
<CUMULATIVE_RUN value="0" /> |
|
43 |
</STATUS> |
|
44 |
|
|
45 |
<SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS> |
|
46 |
</BODY> |
|
47 |
</RESOURCE_PROFILE> |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/schemas/ActionManagerSetDSResourceType.xsd | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" |
|
3 |
elementFormDefault="qualified"> |
|
4 |
<xs:element name="RESOURCE_PROFILE"> |
|
5 |
<xs:complexType> |
|
6 |
<xs:sequence> |
|
7 |
<xs:element name="HEADER"> |
|
8 |
<xs:complexType> |
|
9 |
<xs:sequence> |
|
10 |
<xs:element name="RESOURCE_IDENTIFIER"> |
|
11 |
<xs:complexType> |
|
12 |
<xs:attribute name="value" use="required" /> |
|
13 |
</xs:complexType> |
|
14 |
</xs:element> |
|
15 |
<xs:element name="RESOURCE_TYPE"> |
|
16 |
<xs:complexType> |
|
17 |
<xs:attribute name="value" use="required" type="xs:NCName" /> |
|
18 |
</xs:complexType> |
|
19 |
</xs:element> |
|
20 |
<xs:element name="RESOURCE_KIND"> |
|
21 |
<xs:complexType> |
|
22 |
<xs:attribute name="value" use="required" type="xs:NCName" /> |
|
23 |
</xs:complexType> |
|
24 |
</xs:element> |
|
25 |
<xs:element name="RESOURCE_URI"> |
|
26 |
<xs:complexType> |
|
27 |
<xs:attribute name="value" use="required" type="xs:anyURI" /> |
|
28 |
</xs:complexType> |
|
29 |
</xs:element> |
|
30 |
<xs:element name="DATE_OF_CREATION"> |
|
31 |
<xs:complexType> |
|
32 |
<xs:attribute name="value" use="required" /> |
|
33 |
</xs:complexType> |
|
34 |
</xs:element> |
|
35 |
</xs:sequence> |
|
36 |
</xs:complexType> |
|
37 |
</xs:element> |
|
38 |
<xs:element name="BODY"> |
|
39 |
<xs:complexType> |
|
40 |
<xs:sequence> |
|
41 |
<xs:element name="SET"> |
|
42 |
<xs:complexType> |
|
43 |
<xs:simpleContent> |
|
44 |
<xs:extension base="xs:string"> |
|
45 |
<xs:attribute name="id" use="required" type="xs:string" /> |
|
46 |
<xs:attribute name="directory" use="optional" type="xs:string"/> |
|
47 |
</xs:extension> |
|
48 |
</xs:simpleContent> |
|
49 |
</xs:complexType> |
|
50 |
</xs:element> |
|
51 |
<xs:element name="IMPACT" type="xs:NCName" /> |
|
52 |
<xs:element name="RAW_SETS"> |
|
53 |
<xs:complexType> |
|
54 |
<xs:sequence> |
|
55 |
<xs:element name="LATEST" minOccurs="0" maxOccurs="1" |
|
56 |
type="RAW_SET_TYPE" /> |
|
57 |
<xs:element name="EXPIRED" minOccurs="0" maxOccurs="unbounded" |
|
58 |
type="RAW_SET_TYPE" /> |
|
59 |
</xs:sequence> |
|
60 |
</xs:complexType> |
|
61 |
</xs:element> |
|
62 |
|
|
63 |
<xs:element name="STATUS"> |
|
64 |
<xs:complexType> |
|
65 |
<xs:sequence> |
|
66 |
<xs:element name="LAST_EXECUTION_DATE" type="xs:string" |
|
67 |
nillable="true" /> |
|
68 |
<xs:element name="NUMBER_OF_INFOPACKAGES" type="xs:integer" /> |
|
69 |
<xs:element name="NUMBER_OF_ATOMICACTIONS" type="xs:integer" /> |
|
70 |
</xs:sequence> |
|
71 |
</xs:complexType> |
|
72 |
</xs:element> |
|
73 |
</xs:sequence> |
|
74 |
</xs:complexType> |
|
75 |
</xs:element> |
|
76 |
</xs:sequence> |
|
77 |
</xs:complexType> |
|
78 |
</xs:element> |
|
79 |
|
|
80 |
<xs:complexType name="RAW_SET_TYPE"> |
|
81 |
<xs:attribute name="id" type="xs:string" use="required" /> |
|
82 |
<xs:attribute name="creationDate" type="xs:string" use="required" /> |
|
83 |
<xs:attribute name="lastUpdate" type="xs:string" use="required" /> |
|
84 |
</xs:complexType> |
|
85 |
|
|
86 |
</xs:schema> |
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/actionmanager/applicationContext-dnet-actionmanager-service.properties | ||
---|---|---|
1 |
hbase.actions.table=db_actions |
|
2 |
services.actionmanager.garbage.time.maring=345600000 |
|
3 |
services.actionmanager.hbase.client.startdelay=60000 |
|
4 |
services.actionmanager.basePath=/openaire/actionManager |
|
5 |
services.actionmanager.garbage.retain.threshold=3 |
|
6 |
services.actionmanager.promote.mapper.class=eu.dnetlib.data.mapreduce.hbase.actions2.PromoteActionSetFromHDFSMapper |
|
7 |
|
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/actionmanager/applicationContext-dnet-actionmanager-service.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|
3 |
xmlns:jaxws="http://cxf.apache.org/jaxws" xmlns:sec="http://cxf.apache.org/configuration/security" |
|
4 |
xmlns:wsa="http://cxf.apache.org/ws/addressing" xmlns:p="http://www.springframework.org/schema/p" |
|
5 |
xmlns:http="http://cxf.apache.org/transports/http/configuration" xmlns:t="http://dnetlib.eu/springbeans/t" |
|
6 |
xmlns:template="http://dnetlib.eu/springbeans/template" xmlns:util="http://www.springframework.org/schema/util" |
|
7 |
xmlns="http://www.springframework.org/schema/beans" |
|
8 |
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd |
|
9 |
http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd |
|
10 |
http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd |
|
11 |
http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd |
|
12 |
http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd |
|
13 |
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd |
|
14 |
http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd"> |
|
15 |
|
|
16 |
<bean id="actionManagerService" class="eu.dnetlib.actionmanager.ActionManagerServiceImpl" |
|
17 |
init-method="start" p:notificationHandler-ref="actionManagerNotificationHandler" |
|
18 |
destroy-method="stop" /> |
|
19 |
|
|
20 |
<bean |
|
21 |
id="actionManagerNotificationHandler" |
|
22 |
class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler" |
|
23 |
p:blackboardExecutor-ref="actionManagerBlackboardExecutor" /> |
|
24 |
|
|
25 |
<bean id="actionManagerBlackboardExecutor" |
|
26 |
class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor" |
|
27 |
p:blackboardHandler-ref="blackboardHandler" |
|
28 |
p:actionType="eu.dnetlib.actionmanager.blackboard.ActionManagerActions" |
|
29 |
p:incomplete="false"> |
|
30 |
<property name="actionMap"> |
|
31 |
<map> |
|
32 |
<entry key="GARBAGE_HDFS"> |
|
33 |
<bean class="eu.dnetlib.actionmanager.blackboard.GarbageActionManagerActionFromHDFS"/> |
|
34 |
</entry> |
|
35 |
<entry key="PROMOTE_FROM_HDFS"> |
|
36 |
<bean class="eu.dnetlib.actionmanager.blackboard.PromoteActionManagerActionFromHDFS"/> |
|
37 |
</entry> |
|
38 |
|
|
39 |
</map> |
|
40 |
</property> |
|
41 |
|
|
42 |
</bean> |
|
43 |
|
|
44 |
<!-- M/R Job launcher --> |
|
45 |
<bean id="actionmanagerHbaseJobLauncher" class="eu.dnetlib.actionmanager.blackboard.JobLauncher" |
|
46 |
p:actionTable="${hbase.actions.table}" |
|
47 |
p:dataTable="${hbase.mapred.datatable}" |
|
48 |
p:infomationServiceClient-ref="actionmanagerISClient"/> |
|
49 |
|
|
50 |
<!-- endpoints --> |
|
51 |
<jaxws:endpoint id="actionManagerServiceEndpoint" |
|
52 |
implementor="#actionManagerService" implementorClass="eu.dnetlib.actionmanager.rmi.ActionManagerService" |
|
53 |
address="/actionManager" /> |
|
54 |
|
|
55 |
<template:instance name="serviceRegistrationManager" |
|
56 |
t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl" |
|
57 |
t:name="actionManagerServiceRegistrationManager" t:service="actionManagerService" |
|
58 |
t:endpoint="actionManagerServiceEndpoint" t:jobScheduler="jobScheduler" |
|
59 |
t:serviceRegistrator="actionmanagerServiceRegistrator" /> |
|
60 |
|
|
61 |
<bean id="actionmanagerServiceRegistrator" |
|
62 |
class="eu.dnetlib.enabling.tools.registration.BlackboardServiceRegistrator" |
|
63 |
p:serviceLocator-ref="uniqueServiceLocator" |
|
64 |
p:serviceNameResolver-ref="defaultServiceNameResolver" |
|
65 |
p:hnmLocator-ref="${services.registration.default.hnmlocator}" |
|
66 |
p:eprBuilder-ref="jaxwsEndpointReferenceBuilder"> |
|
67 |
<property name="serviceProperties"> |
|
68 |
<map> |
|
69 |
<entry key="garbageRetainThreshold" value="${services.actionmanager.garbage.retain.threshold}"/> |
|
70 |
<entry key="garbageTimeMargin" value="${services.actionmanager.garbage.time.maring}" /> |
|
71 |
<entry key="basePath" value="${services.actionmanager.basePath}" /> |
|
72 |
</map> |
|
73 |
</property> |
|
74 |
</bean> |
|
75 |
|
|
76 |
<!-- Information Service Client --> |
|
77 |
<bean id="actionmanagerISClient" class="eu.dnetlib.actionmanager.is.ISClient" |
|
78 |
p:actionManagerSetDsTemplate-ref="actionManagerSetDsTemplate" |
|
79 |
p:endpoint-ref="actionManagerServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" /> |
|
80 |
|
|
81 |
<bean id="actionManagerSetDsTemplate" |
|
82 |
class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory" |
|
83 |
p:template="classpath:/eu/dnetlib/actionmanager/tmpl/actionManagerSetDS.xml.tmpl" |
|
84 |
scope="prototype" /> |
|
85 |
|
|
86 |
|
|
87 |
</beans> |
Also available in: Unified diff
[maven-release-plugin] copy for tag dnet-actionmanager-service-5.2.1