Project

General

Profile

« Previous | Next » 

Revision 48014

[maven-release-plugin] copy for tag dnet-actionmanager-service-5.2.1

View differences:

modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-actionmanager-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-actionmanager-service"}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/java/eu/dnetlib/actionmanager/xslt/RelationXsltFunctionsTest.java
1
package eu.dnetlib.actionmanager.xslt;
2

  
3
import java.io.StringReader;
4
import java.io.StringWriter;
5

  
6
import javax.xml.transform.Source;
7
import javax.xml.transform.Transformer;
8
import javax.xml.transform.TransformerFactory;
9
import javax.xml.transform.stream.StreamResult;
10
import javax.xml.transform.stream.StreamSource;
11

  
12
import org.junit.Before;
13
import org.junit.Test;
14

  
15
public class RelationXsltFunctionsTest {
16
	private Transformer transformer;
17

  
18
	@Before
19
	public void setUp() throws Exception {
20
		Source source = new StreamSource(getClass().getResourceAsStream("/eu/dnetlib/actionmanager/xslt/rels2actions.xslt"));
21
		transformer = TransformerFactory.newInstance().newTransformer(source);
22
	}
23

  
24
	@Test
25
	public void test_01() throws Exception {
26
		StringBuffer xml = new StringBuffer();
27
		xml.append("<RELATIONS>");
28
		xml.append("</RELATIONS>");
29

  
30
		applyXslt(xml.toString());
31
	}
32

  
33
	@Test
34
	public void test_02() throws Exception {
35
		StringBuffer xml = new StringBuffer();
36
		xml.append("<RELATIONS>");
37
		xml.append("<RELATION type='resultProject' source='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' target='corda_______::281877' />");
38
		xml.append("<RELATION type='resultProject' source='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' target='40|corda_______::0001fc1603869410927ddbde3afb2403' />");
39
		xml.append("<RELATION type='personResult' source='30|crossref____::000178abc5a314472603976a1be68927' target='50|crossref____::7fb616c8cdd08f0f12a04ac0f18edf80' />");
40
		xml.append("</RELATIONS>");
41

  
42
		applyXslt(xml.toString());
43
	}
44

  
45
	public String applyXslt(String xml) throws Exception {
46
		final Source xmlSource = new StreamSource(new StringReader(xml));
47
		final StringWriter output = new StringWriter();
48

  
49
		transformer.transform(xmlSource, new StreamResult(output));
50

  
51
		System.out.println(output.toString());
52

  
53
		return output.toString();
54
	}
55

  
56
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/resources/eu/dnetlib/actionmanager/test/applicationContext-actionmanager-test.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns="http://www.springframework.org/schema/beans"
3
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
4
	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
5
	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
6
	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
7
	xmlns:util="http://www.springframework.org/schema/util"
8
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
9
                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
10
                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
11
                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
12
                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
13
                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
14
                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
15

  
16
	<bean id="hbaseActionManagerCore" class="eu.dnetlib.actionmanager.hbase.HBaseActionManagerCore"
17
		p:actionFactory-ref="actionFactory"
18
		p:hbaseClient-ref="hbaseClient" />
19
		
20
	<bean id="hbaseClient" class="eu.dnetlib.actionmanager.hbase.HBaseClient"
21
		p:tableName="db_actions_michele"
22
		p:hbaseConf="${services.data.hbase.dm.properties}" 
23
		init-method="initWithEmptyTable" />		
24
		<!-- p:hbaseConf="classpath:/eu/dnetlib/actionmanager/test/hbase.properties" -->
25
		
26
	<bean id="actionFactory" class="eu.dnetlib.actionmanager.actions.ActionFactory">
27
		<property name="xslts">
28
			<util:map>
29
				<entry key="dmf2actions" value="classpath:/eu/dnetlib/actionmanager/xslt/dmf2actions.xslt" />
30
			</util:map>
31
		</property>
32
	</bean>
33

  
34

  
35
</beans>
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/test/resources/eu/dnetlib/actionmanager/test/hbase.properties
1
#COMMON
2
hadoop.entry.node                               =       node1.hadoop.research-infrastructures.eu
3

  
4
#CORE-SITE
5
fs.defaultFS                                    =       hdfs://${hadoop.entry.node}:8020
6
io.file.buffer.size                     =       65536
7
io.compression.codecs                   =       org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec
8
hadoop.security.authentication  =       simple
9
hadoop.security.auth_to_local   =       DEFAULT
10

  
11
#HBASE-SITE
12
hbase.rootdir                                   =       ${fs.defaultFS}/hbase
13
hbase.client.write.buffer               =       2097152
14
hbase.client.pause                              =       1000
15
hbase.client.retries.number             =       10
16
hbase.client.scanner.caching    =       1
17
hbase.client.keyvalue.maxsize   =       10485760
18
hbase.security.authentication   =       simple
19
zookeeper.session.timeout               =       60000
20
zookeeper.znode.parent                  =       /hbase
21
zookeeper.znode.rootserver              =       root-region-server
22
hbase.zookeeper.quorum                  =       ${hadoop.entry.node}
23
hbase.zookeeper.property.clientPort     =       2182
24

  
25
#HDFS-SITE
26
dfs.https.port                                  =       50470
27
dfs.namenode.https-address              =       ${hadoop.entry.node}:${dfs.https.port}
28
dfs.namenode.http-address               =       ${hadoop.entry.node}:50070
29
dfs.replication                                 =       3
30
dfs.blocksize                                   =       134217728
31
dfs.client.use.datanode.hostname=       false
32

  
33
#MAPRED-SITE
34
mapreduce.jobtracker.address    =       ${hadoop.entry.node}:8021
35
mapred.job.tracker                              =       ${hadoop.entry.node}:8021
36
mapred.mapper.new-api                   =       true
37
mapred.reducer.new-api                  =       true
38
mapred.output.compress                  =       false
39
mapred.output.compression.type  =       BLOCK
40
mapred.output.compression.codec =       org.apache.hadoop.io.compress.DefaultCodec
41
#mapred.map.output.compression.codec    =       org.apache.hadoop.io.compress.SnappyCodec
42
mapred.compress.map.output              =       true
43
#io.sort.factor                                 =       64
44
#io.sort.record.percent                 =       0.05
45
#io.sort.spill.percent                  =       0.8
46
#mapred.reduce.parallel.copies  =       10
47
#mapred.submit.replication              =       1
48
#mapred.reduce.tasks                            =       8
49
#io.sort.mb                                             =       56
50
#mapred.child.java.opts                 =        -Xmx238449165
51
#mapred.job.reuse.jvm.num.tasks =       1
52
#mapred.map.tasks                               =       6
53
#mapred.map.tasks.speculative.execution =       false
54
#mapred.reduce.tasks.speculative.execution      =       false
55
#mapred.reduce.slowstart.completed.maps =       0.8
56
##mapred.local.dir                              = /data/mapred/local,/data/mapred/jt
57
mapred.local.dir                                = /tmp
58
mapred.child.java.opts                  = -Xmx512973352
59

  
60
#dfs.client.use.legacy.blockreader=true
61
 
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/BlackboardJobListener.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import eu.dnetlib.actionmanager.rmi.ActionManagerException;
4
import eu.dnetlib.enabling.tools.blackboard.AbstractBlackboardJobListener;
5
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
6
import org.apache.commons.logging.Log;
7
import org.apache.commons.logging.LogFactory;
8

  
9
public class BlackboardJobListener extends AbstractBlackboardJobListener {
10

  
11
	private static final Log log = LogFactory.getLog(BlackboardJobListener.class);
12

  
13
	private JobCallback callback;
14

  
15
	public BlackboardJobListener(final JobCallback callback) {
16
		super();
17
		this.callback = callback;
18
	}
19

  
20
	@Override
21
	protected void onDone(final BlackboardJob job) {
22
		try {
23
			if (callback != null) {
24
				callback.onDone(job.getParameters());
25
			}
26
		} catch (Exception e) {
27
			log.error("fail to run a callback function");
28
		}
29
	}
30

  
31
	@Override
32
	protected void onFailed(final BlackboardJob job) {
33
		try {
34
			if (callback != null) {
35
				callback.onFailed(new ActionManagerException(job.getError()));
36
			}
37
		} catch (Exception e) {
38
			log.error("fail to run a callback function");
39
		}
40
	}
41
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/PromoteActionManagerActionFromHDFS.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import java.util.Map;
4
import java.util.Set;
5

  
6
import com.google.common.base.Splitter;
7
import com.google.common.collect.Sets;
8
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
9
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
10
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
11
import org.apache.commons.lang.StringUtils;
12

  
13
public class PromoteActionManagerActionFromHDFS extends AbstractActionManagerAction implements BlackboardServerAction<ActionManagerActions> {
14

  
15
	@Override
16
	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
17

  
18
		final String setParam = job.getParameters().get("set");
19

  
20
		Set<String> sets = null;
21
		if (StringUtils.isNotBlank(setParam) && !setParam.equalsIgnoreCase(JobLauncher.ALL_SETS)) {
22
			sets = Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(setParam));
23
		}
24

  
25
		try {
26
			getHbaseJobLauncher().executePromoteFromHDFSJob(sets, new JobCallback() {
27

  
28
				@Override
29
				public void onDone(final Map<String, String> properties) {
30
					job.getParameters().putAll(properties);
31
					handler.done(job);
32
				}
33

  
34
				@Override
35
				public void onFailed(final Throwable e) {
36
					handler.failed(job, e);
37
				}
38

  
39
			});
40
		} catch (Throwable e) {
41
			handler.failed(job, e);
42
		}
43

  
44
	}
45
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/JobCallback.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import java.util.Map;
4

  
5
public interface JobCallback {
6

  
7
	void onDone(Map<String, String> properties);
8

  
9
	void onFailed(Throwable e);
10

  
11
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/GarbageActionManagerActionFromHDFS.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
4
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
5
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
6

  
7
public class GarbageActionManagerActionFromHDFS extends AbstractActionManagerAction implements BlackboardServerAction<ActionManagerActions> {
8

  
9
	@Override
10
	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
11
		try {
12
			getHbaseJobLauncher().executeGarbageActions();
13
			handler.done(job);
14
		} catch (Throwable e) {
15
			handler.failed(job, e);
16
		}
17
	}
18
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/ActionManagerActions.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
public enum ActionManagerActions {
4
	PROMOTE_FROM_HDFS, GARBAGE_HDFS
5
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/AbstractActionManagerAction.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import javax.annotation.Resource;
4

  
5
public class AbstractActionManagerAction {
6

  
7
	@Resource
8
	private JobLauncher jobLauncher;
9

  
10
	public JobLauncher getHbaseJobLauncher() {
11
		return jobLauncher;
12
	}
13

  
14
	public void setJobLauncher(final JobLauncher jobLauncher) {
15
		this.jobLauncher = jobLauncher;
16
	}
17

  
18
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/blackboard/JobLauncher.java
1
package eu.dnetlib.actionmanager.blackboard;
2

  
3
import java.text.ParseException;
4
import java.util.Date;
5
import java.util.List;
6
import java.util.Map;
7
import java.util.Set;
8
import java.util.concurrent.Executor;
9
import java.util.concurrent.Executors;
10
import javax.annotation.Resource;
11

  
12
import com.google.common.base.Function;
13
import com.google.common.base.Joiner;
14
import com.google.common.base.Predicate;
15
import com.google.common.collect.Iterables;
16
import com.google.common.collect.Lists;
17
import com.google.common.collect.Maps;
18
import eu.dnetlib.actionmanager.is.ISClient;
19
import eu.dnetlib.actionmanager.rmi.ActionManagerException;
20
import eu.dnetlib.actionmanager.set.ActionManagerSet;
21
import eu.dnetlib.actionmanager.set.RawSet;
22
import eu.dnetlib.data.hadoop.config.ClusterName;
23
import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions;
24
import eu.dnetlib.data.hadoop.rmi.HadoopService;
25
import eu.dnetlib.data.hadoop.rmi.HadoopServiceException;
26
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
27
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
28
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
29
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
30
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
31
import eu.dnetlib.enabling.tools.blackboard.BlackboardClientHandler;
32
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
33
import eu.dnetlib.enabling.tools.blackboard.BlackboardJobRegistry;
34
import eu.dnetlib.miscutils.datetime.DateUtils;
35
import org.apache.commons.logging.Log;
36
import org.apache.commons.logging.LogFactory;
37
import org.springframework.beans.factory.annotation.Autowired;
38
import org.springframework.beans.factory.annotation.Required;
39
import org.springframework.beans.factory.annotation.Value;
40

  
41
public class JobLauncher {
42

  
43
	public static final String ALL_SETS = "__ALL__";
44
	public static final String SEMICOLON = ";";
45
	public static final String COLON = ",";
46
	public static final String SEQFILE_INPUTFORMAT = "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat";
47
	private final static String UPDATE_ACTION_PROFILE = "for $x in "
48
			+ "/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'ActionManagerSetDSResourceType' and .//SET/@id = '%s'] "
49
			+ "return update delete $x//RAW_SETS/EXPIRED[@id = '%s']";
50
	private static final Log log = LogFactory.getLog(JobLauncher.class); // NOPMD by marko on 11/24/08 5:02 PM
51
	private final Executor executor = Executors.newCachedThreadPool();
52

  
53
	@Resource
54
	private UniqueServiceLocator serviceLocator;
55

  
56
	private ISClient infomationServiceClient;
57

  
58
	/**
59
	 * blackboard handler.
60
	 */
61
	@Resource
62
	private BlackboardClientHandler blackboardClientHandler;
63

  
64
	/**
65
	 * blackboard job registry.
66
	 */
67
	@Resource
68
	private BlackboardJobRegistry jobRegistry;
69

  
70
	@Autowired
71
	private ISClient isClient;
72

  
73
	@Value("${services.actionmanager.promote.mapper.class}")
74
	private String promoteJobMapperClass;
75

  
76
	private String actionTable;
77
	private String dataTable;
78

  
79
	public void executePromoteFromHDFSJob(final Set<String> sets, final JobCallback callback) throws ActionManagerException, ISLookUpException {
80
		log.info("Starting commit m/r job; sets=" + sets);
81

  
82
		final List<String> paths = Lists.newArrayList();
83
		if (sets == null || sets.isEmpty()) {
84

  
85
			for (ActionManagerSet set : infomationServiceClient.listSets()) {
86
				paths.add(set.getPathToLatest());
87
			}
88

  
89
		} else {
90
			for (String setId : sets) {
91
				if (infomationServiceClient.existsSet(setId)) {
92
					final ActionManagerSet set = infomationServiceClient.getSet(setId);
93
					paths.add(set.getPathToLatest());
94
				} else {
95
					log.error("Invalid set " + setId);
96
					throw new ActionManagerException("Invalid set " + setId);
97
				}
98
			}
99
		}
100

  
101
		final HadoopService hadoopService = serviceLocator.getService(HadoopService.class);
102
		final List<String> existingPaths = Lists.newArrayList(Iterables.filter(paths, new Predicate<String>(){
103
			@Override
104
			public boolean apply(final String path) {
105

  
106
				try {
107
					final boolean exist = hadoopService.existHdfsPath(ClusterName.DM.toString(), path);
108
					if (!exist) {
109
						log.warn(String.format("path '%s' doesn't exist on DM cluster'", path));
110
					}
111
					return exist;
112
				} catch (HadoopServiceException e) {
113
					log.error(e);
114
					return false;
115
				}
116
			}
117
		}));
118

  
119
		final Map<String, String> params = Maps.newHashMap();
120
		params.put("mapred.input.dir.formats", Joiner.on(COLON).join(Iterables.transform(existingPaths, new Function<String, String>() {
121
			@Override
122
			public String apply(final String path) {
123
				return path + SEMICOLON + SEQFILE_INPUTFORMAT;
124
			}
125
		})));
126
		params.put("mapred.input.dir.mappers", Joiner.on(COLON).join(Iterables.transform(existingPaths, new Function<String, String>() {
127
			@Override
128
			public String apply(final String path) {
129
				return path + SEMICOLON + promoteJobMapperClass;
130
			}
131
		})));
132

  
133
		params.put("hbase.mapred.outputtable", dataTable);
134
		params.put("hbase.mapreduce.outputtable", dataTable);
135

  
136
		log.info("promoting HDFS rawsets: " + existingPaths);
137

  
138
		executeHDFS("promoteMultipleActionSetsJob", params, callback);
139
	}
140

  
141
	private void executeHDFS(final String jobName, final Map<String, String> params, final JobCallback callback) throws ActionManagerException {
142

  
143
		if (params == null || params.isEmpty()) { throw new ActionManagerException("Missing HDFS paths"); }
144

  
145
		executor.execute(new Runnable() {
146

  
147
			@Override
148
			public void run() {
149
				try {
150
					final String serviceId = findHadoopServiceProfileID();
151
					final BlackboardJob bbJob = blackboardClientHandler.newJob(serviceId);
152

  
153
					bbJob.setAction(HadoopBlackboardActions.SUBMIT_MAPREDUCE_JOB.toString());
154
					bbJob.getParameters().put("job.name", jobName);
155
					bbJob.getParameters().put("cluster", ClusterName.DM.toString());
156
					bbJob.getParameters().putAll(params);
157

  
158
					jobRegistry.registerJobListener(bbJob, new BlackboardJobListener(callback));
159
					blackboardClientHandler.assign(bbJob);
160
				} catch (Exception e) {
161
					log.error("Error launching m/r job: " + jobName, e);
162
					throw new RuntimeException("Error launching m/r job: " + jobName, e);
163
				}
164
			}
165

  
166
		});
167
	}
168

  
169
	public void executeGarbageActions() throws ISRegistryException, HadoopServiceException, ParseException, ActionManagerException {
170
		final String basePath = isClient.getBasePathHDFS();
171
		final Integer garbageRetain = Integer.valueOf(isClient.getGarbageRetainThreshold());
172
		final Long dateLimit = DateUtils.now() - Long.valueOf(isClient.getGarbageTimeMargin());
173

  
174
		final HadoopService hadoopService = serviceLocator.getService(HadoopService.class);
175
		final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class);
176

  
177
		for (ActionManagerSet set : isClient.listValidSets()) {
178
			for (RawSet expired : Iterables.limit(set.getExpired(), garbageRetain)) {
179
				final Date d = org.apache.commons.lang.time.DateUtils.parseDate(expired.getLastUpdate(), ActionManagerSet.DATE_PATTERNS);
180
				if (d.getTime() < dateLimit) {
181

  
182
					log.info(String.format("removing raw action set %s/%s", set.getId(), expired.getId()));
183
					hadoopService.deleteHdfsPath(ClusterName.DM.toString(), basePath + "/" + set.getDirectory() + "/" + expired.getId());
184

  
185
					final String xUpdate = String.format(UPDATE_ACTION_PROFILE, set.getId(), expired.getId());
186
					log.info(String.format("updating ActionSet profile: %s", xUpdate));
187
					isRegistry.executeXUpdate(xUpdate);
188
				}
189
			}
190
		}
191
	}
192

  
193
	private String findHadoopServiceProfileID() throws Exception {
194
		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
195
				"collection('/db/DRIVER/ServiceResources/HadoopServiceResourceType')//RESOURCE_IDENTIFIER/@value/string()");
196
	}
197

  
198
	public String getActionTable() {
199
		return actionTable;
200
	}
201

  
202
	@Required
203
	public void setActionTable(final String actionTable) {
204
		this.actionTable = actionTable;
205
	}
206

  
207
	public String getDataTable() {
208
		return dataTable;
209
	}
210

  
211
	@Required
212
	public void setDataTable(final String dataTable) {
213
		this.dataTable = dataTable;
214
	}
215

  
216
	public ISClient getInfomationServiceClient() {
217
		return infomationServiceClient;
218
	}
219

  
220
	@Required
221
	public void setInfomationServiceClient(final ISClient infomationServiceClient) {
222
		this.infomationServiceClient = infomationServiceClient;
223
	}
224

  
225
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/inspector/ActionManagerController.java
1
package eu.dnetlib.actionmanager.inspector;
2

  
3
import java.util.List;
4
import java.util.Set;
5

  
6
import eu.dnetlib.actionmanager.ActionManagerConstants.ACTION_TYPE;
7
import eu.dnetlib.actionmanager.blackboard.JobLauncher;
8
import eu.dnetlib.actionmanager.common.Provenance;
9
import eu.dnetlib.actionmanager.is.ISClient;
10
import eu.dnetlib.actionmanager.rmi.ActionManagerException;
11
import eu.dnetlib.actionmanager.set.ActionManagerSet;
12
import eu.dnetlib.enabling.inspector.AbstractInspectorController;
13
import org.apache.commons.lang.StringUtils;
14
import org.springframework.beans.factory.annotation.Autowired;
15
import org.springframework.stereotype.Controller;
16
import org.springframework.ui.ModelMap;
17
import org.springframework.web.bind.annotation.ModelAttribute;
18
import org.springframework.web.bind.annotation.RequestMapping;
19
import org.springframework.web.bind.annotation.RequestParam;
20

  
21
@Controller
22
public class ActionManagerController extends AbstractInspectorController {
23

  
24
	@Autowired
25
	private ISClient isClient;
26

  
27
	@Autowired
28
	private JobLauncher jobLauncher;
29

  
30
	@ModelAttribute(value = "types")
31
	public ACTION_TYPE[] listActionTypes() {
32
		return ACTION_TYPE.values();
33
	}
34

  
35
	@ModelAttribute(value = "provenances")
36
	public Provenance[] listProvenances() {
37
		return Provenance.values();
38
	}
39

  
40
	@ModelAttribute(value = "sets")
41
	public List<ActionManagerSet> listSets() throws ActionManagerException {
42
		return isClient.listSets();
43
	}
44

  
45
	@RequestMapping("/inspector/actions/createSet.do")
46
	public String createSet(final ModelMap map,
47
			@RequestParam(value = "id", required = true) final String id,
48
			@RequestParam(value = "name", required = true) final String name,
49
			@RequestParam(value = "directory", required = true) final String directory) throws Exception {
50
		isClient.registerSetProfile(new ActionManagerSet(id, name, directory));
51
		return "redirect:listSets.do";
52
	}
53

  
54
	@RequestMapping("/inspector/actions/listSets.do")
55
	public void createSet(final ModelMap map) throws Exception {
56
		map.addAttribute("sets", isClient.listValidSets());
57
	}
58

  
59
	@RequestMapping("/inspector/actions/commit.do")
60
	public void commit(final ModelMap map, @RequestParam(value = "sets", required = false) final Set<String> sets) throws Exception {
61
		jobLauncher.executePromoteFromHDFSJob(sets, null);
62
		map.addAttribute("message", "Job launched !!!");
63
	}
64

  
65
	@RequestMapping("/inspector/actions/garbage.do")
66
	public void garbage(final ModelMap map) throws Exception {
67
		jobLauncher.executeGarbageActions();
68
		map.addAttribute("message", "Job launched !!!");
69
	}
70

  
71
	@RequestMapping("/inspector/actions/deleteSet.do")
72
	public String deleteSet(final ModelMap map, @RequestParam(value = "set", required = true) final String set) throws Exception {
73
		map.addAttribute("allSets", JobLauncher.ALL_SETS);
74
		if (set != null && !set.isEmpty()) {
75
			isClient.deleteSetProfile(set);
76
		}
77
		return "redirect:listSets.do";
78
	}
79

  
80
	@RequestMapping("/inspector/actions/linkActionSetProfile.do")
81
	public String linkProfile(@RequestParam(value = "set", required = true) final String set) throws Exception {
82
		if (StringUtils.isBlank(set)) {
83
			throw new IllegalArgumentException("missing set id");
84
		}
85
		final String profileId = isClient.getProfileId(set);
86
		return "redirect:/mvc/inspector/index.do/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType/" + profileId + "/show";
87
	}
88

  
89
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/ActionManagerServiceImpl.java
1
package eu.dnetlib.actionmanager;
2

  
3
import java.util.List;
4
import javax.jws.WebParam;
5

  
6
import eu.dnetlib.actionmanager.is.ISClient;
7
import eu.dnetlib.actionmanager.rmi.ActionManagerException;
8
import eu.dnetlib.actionmanager.rmi.ActionManagerService;
9
import eu.dnetlib.actionmanager.set.ActionManagerSet;
10
import eu.dnetlib.data.hadoop.config.ClusterName;
11
import eu.dnetlib.data.hadoop.rmi.HadoopService;
12
import eu.dnetlib.data.hadoop.rmi.HadoopServiceException;
13
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
14
import eu.dnetlib.enabling.tools.AbstractBaseService;
15
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
16
import org.apache.hadoop.fs.Path;
17
import org.springframework.beans.factory.annotation.Autowired;
18
import org.springframework.beans.factory.annotation.Required;
19

  
20
public class ActionManagerServiceImpl extends AbstractBaseService implements ActionManagerService {
21

  
22
	private NotificationHandler notificationHandler;
23

  
24
	@Autowired
25
	private ISClient isClient;
26

  
27
	@Autowired
28
	private UniqueServiceLocator serviceLocator;
29

  
30
	@Override
31
	public String createSet(@WebParam(name = "set") final ActionManagerSet set) throws ActionManagerException {
32
		try {
33
			final String basePath = isClient.getBasePathHDFS();
34
			final Path actionSetPath = new Path(basePath + "/" + set.getId());
35
			serviceLocator.getService(HadoopService.class).createHdfsDirectory(ClusterName.DM.toString(), actionSetPath.toString(), true);
36
			return isClient.registerSetProfile(set);
37
		} catch (HadoopServiceException e) {
38
			throw new ActionManagerException(e);
39
		}
40
	}
41

  
42
	@Override
43
	public boolean deleteSet(@WebParam(name = "set") final String setId) throws ActionManagerException {
44
		try {
45
			final String basePath = isClient.getBasePathHDFS();
46
			final Path actionSetPath = new Path(basePath + "/" + setId);
47
			serviceLocator.getService(HadoopService.class).deleteHdfsPath(ClusterName.DM.toString(), actionSetPath.toString());
48
			return isClient.deleteSetProfile(setId);
49
		} catch (HadoopServiceException e) {
50
			throw new ActionManagerException(e);
51
		}
52
	}
53

  
54
	@Override
55
	public List<ActionManagerSet> ListSets() throws ActionManagerException {
56
		return isClient.listValidSets();
57
	}
58

  
59
	@Override
60
	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
61
		notificationHandler.notified(subscriptionId, topic, isId, message);
62
	}
63

  
64
	public NotificationHandler getNotificationHandler() {
65
		return notificationHandler;
66
	}
67

  
68
	@Required
69
	public void setNotificationHandler(final NotificationHandler notificationHandler) {
70
		this.notificationHandler = notificationHandler;
71
	}
72

  
73
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/java/eu/dnetlib/actionmanager/is/ISClient.java
1
package eu.dnetlib.actionmanager.is;
2

  
3
import java.io.StringReader;
4
import java.util.List;
5
import java.util.NoSuchElementException;
6
import java.util.Set;
7
import javax.annotation.Resource;
8
import javax.xml.ws.Endpoint;
9

  
10
import com.google.common.collect.Iterables;
11
import com.google.common.collect.Lists;
12
import com.google.common.collect.Sets;
13
import eu.dnetlib.actionmanager.rmi.ActionManagerException;
14
import eu.dnetlib.actionmanager.set.ActionManagerSet;
15
import eu.dnetlib.actionmanager.set.ActionManagerSet.ImpactTypes;
16
import eu.dnetlib.actionmanager.set.RawSet;
17
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
18
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
19
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
20
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
21
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
22
import eu.dnetlib.miscutils.datetime.DateUtils;
23
import eu.dnetlib.soap.EndpointReferenceBuilder;
24
import org.antlr.stringtemplate.StringTemplate;
25
import org.apache.commons.logging.Log;
26
import org.apache.commons.logging.LogFactory;
27
import org.dom4j.Attribute;
28
import org.dom4j.Document;
29
import org.dom4j.Element;
30
import org.dom4j.io.SAXReader;
31
import org.springframework.beans.factory.annotation.Required;
32

  
33
public class ISClient {
34

  
35
	private static final Log log = LogFactory.getLog(ISClient.class); // NOPMD by marko on 11/24/08 5:02 PM
36
	@Resource
37
	private UniqueServiceLocator serviceLocator;
38
	/**
39
	 * Template for ActionManagerSet profiles
40
	 */
41
	private StringTemplate actionManagerSetDsTemplate;
42
	/**
43
	 * service endpoint.
44
	 */
45
	private Endpoint endpoint;
46
	/**
47
	 * endpoint builder.
48
	 */
49
	private EndpointReferenceBuilder<Endpoint> eprBuilder;
50

  
51
	public String registerSetProfile(final ActionManagerSet set) throws ActionManagerException {
52
		if (existsSet(set.getId())) { throw new ActionManagerException("Set " + set.getId() + " already registered"); }
53
		try {
54
			StringTemplate template = new StringTemplate(actionManagerSetDsTemplate.getTemplate());
55
			template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
56
			template.setAttribute("set", set);
57
			template.setAttribute("latest", RawSet.newInstance());
58
			return serviceLocator.getService(ISRegistryService.class).registerProfile(template.toString());
59
		} catch (ISRegistryException e) {
60
			throw new ActionManagerException("Error registering set " + set, e);
61
		}
62
	}
63

  
64
	public List<ActionManagerSet> listSets() throws ActionManagerException {
65
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') return $x";
66
		return getActionManagerSets(q);
67
	}
68

  
69
	public List<ActionManagerSet> listValidSets() throws ActionManagerException {
70
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') \n"
71
				+ "where string-length($x//RAW_SETS/LATEST/@id/string()) > 0\n"
72
				+ "return $x";
73

  
74
		return getActionManagerSets(q);
75
	}
76

  
77
	private List<ActionManagerSet> getActionManagerSets(final String q) throws ActionManagerException {
78
		final List<ActionManagerSet> list = Lists.newArrayList();
79

  
80
		try {
81
			final String basePath = getBasePathHDFS();
82
			for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q)) {
83
				list.add(getActionManagerSet(basePath, s));
84
			}
85
		} catch (ISLookUpException e) {
86
			log.error("Error accessing Sets, using query: " + q);
87
		}
88
		return list;
89
	}
90

  
91
	public ActionManagerSet getSet(final String setId) throws ActionManagerException, ISLookUpException {
92
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
93
				+ "where $x//SET/@id = '" + setId + "' return $x";
94

  
95
		try {
96
			final String basePath = getBasePathHDFS();
97
			final String setProfile = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q);
98
			return getActionManagerSet(basePath, setProfile);
99
		} catch (ISLookUpException e) {
100
			throw new ActionManagerException("Error accessing Sets, using query: " + q);
101
		}
102
	}
103

  
104
	private ActionManagerSet getActionManagerSet(final String basePath, final String profile) throws ActionManagerException {
105
		final SAXReader reader = new SAXReader();
106
		final ActionManagerSet set = new ActionManagerSet();
107

  
108
		try {
109
			final Document doc = reader.read(new StringReader(profile));
110

  
111
			set.setId(doc.valueOf("//SET/@id").trim());
112
			set.setName(doc.valueOf("//SET").trim());
113
			set.setImpact(ImpactTypes.valueOf(doc.valueOf("//IMPACT").trim()));
114
			set.setLatest(doc.valueOf("//RAW_SETS/LATEST/@id"), doc.valueOf("//RAW_SETS/LATEST/@creationDate"), doc.valueOf("//RAW_SETS/LATEST/@lastUpdate"));
115
			set.setDirectory(doc.valueOf("//SET/@directory"));
116
			final List expiredNodes = doc.selectNodes("//RAW_SETS/EXPIRED");
117
			if (expiredNodes != null) {
118
				for (int i = 0; i < expiredNodes.size(); i++) {
119
					Element ex = (Element) expiredNodes.get(i);
120
					set.addExpired(ex.attributeValue("id"), ex.attributeValue("creationDate"), ex.attributeValue("lastUpdate"));
121
				}
122
			}
123

  
124
			final StringBuilder sb = new StringBuilder();
125
			sb.append(basePath);
126
			sb.append("/");
127
			sb.append(doc.valueOf("//SET/@directory"));
128
			sb.append("/");
129
			sb.append(doc.valueOf("//RAW_SETS/LATEST/@id"));
130
			set.setPathToLatest(sb.toString());
131

  
132
			return set;
133
		} catch (Exception e) {
134
			throw new ActionManagerException("Error creating set from profile: " + profile, e);
135
		}
136
	}
137

  
138
	public boolean deleteSetProfile(final String setId) throws ActionManagerException {
139
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + setId
140
				+ "' return $x//RESOURCE_IDENTIFIER/@value/string()";
141

  
142
		try {
143
			final String profId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q);
144
			return serviceLocator.getService(ISRegistryService.class).deleteProfile(profId);
145
		} catch (Exception e) {
146
			log.error("Error deleting set " + setId, e);
147
			throw new ActionManagerException("Error deleting set " + setId, e);
148
		}
149
	}
150

  
151
	public boolean existsSet(final String id) throws ActionManagerException {
152
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + id
153
				+ "' return $x";
154
		try {
155
			return !serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q).isEmpty();
156
		} catch (ISLookUpException e) {
157
			log.error("Error accessing Sets, using query: " + q, e);
158
			throw new ActionManagerException("Error running xquery: " + q, e);
159
		}
160
	}
161

  
162
	public boolean existsRawSet(final String id) throws ActionManagerException {
163
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//RAW_SETS/*/@id = '" + id
164
				+ "' return $x";
165
		try {
166
			return !serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q).isEmpty();
167
		} catch (ISLookUpException e) {
168
			log.error("Error accessing RawSets, using query: " + q, e);
169
			throw new ActionManagerException("Error running xquery: " + q, e);
170
		}
171
	}
172

  
173
	public Set<String> listValidRawSets() throws ActionManagerException {
174
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') return $x//RAW_SETS/LATEST/@id/string()";
175
		try {
176
			final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q);
177
			return Sets.newHashSet(list);
178
		} catch (Exception e) {
179
			log.error("Error running xquery: " + q, e);
180
			throw new ActionManagerException("Error running xquery: " + q, e);
181
		}
182
	}
183

  
184
	public RawSet geLatestRawSet(final String set) throws ActionManagerException {
185
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set
186
				+ "' return concat(' ' , $x//RAW_SETS/LATEST/@id, ' @@@ ', $x//RAW_SETS/LATEST/@creationDate, ' @@@ ', $x//RAW_SETS/LATEST/@lastUpdate, ' ')";
187
		try {
188
			String[] arr = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q).split("@@@");
189
			return RawSet.newInstance(arr[0].trim(), arr[1].trim(), arr[2].trim());
190
		} catch (ISLookUpException e) {
191
			log.error("Error accessing Sets, using query: " + q, e);
192
			throw new ActionManagerException("Error running xquery: " + q, e);
193
		}
194
	}
195

  
196
	public String getGarbageTimeMargin() throws ActionManagerException {
197
		return queryServiceProperty("garbageTimeMargin");
198
	}
199

  
200
	public String getBasePathHDFS() throws ActionManagerException {
201
		return queryServiceProperty("basePath");
202
	}
203

  
204
	public String getGarbageRetainThreshold() throws ActionManagerException {
205
		return queryServiceProperty("garbageRetainThreshold");
206
	}
207

  
208
	private String queryServiceProperty(final String propertyName) throws ActionManagerException {
209
		final String q = "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
210
				+ propertyName + "']/@value/string()";
211
		log.debug("quering for service property: " + q);
212
		try {
213
			final List<String> value = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(q);
214
			return Iterables.getOnlyElement(value);
215
		} catch (ISLookUpException e) {
216
			String msg = "Error accessing service profile, using query: " + q;
217
			log.error(msg, e);
218
			throw new ActionManagerException(msg, e);
219
		} catch (NoSuchElementException e) {
220
			String msg = "missing service property: " + propertyName;
221
			log.error(msg, e);
222
			throw new ActionManagerException(msg, e);
223
		} catch (IllegalArgumentException e) {
224
			String msg = "found more than one service property: " + propertyName;
225
			log.error(msg, e);
226
			throw new ActionManagerException(msg, e);
227
		}
228
	}
229

  
230
	public void addLatestRawSet(final String set, final RawSet rawSet) throws ActionManagerException {
231
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set
232
				+ "' return $x";
233
		try {
234
			final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q);
235
			final Document doc = new SAXReader().read(new StringReader(profile));
236
			final String profId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
237
			final Element latest = (Element) doc.selectSingleNode("//RAW_SETS/LATEST");
238
			final Element expired = ((Element) doc.selectSingleNode("//RAW_SETS")).addElement("EXPIRED");
239

  
240
			for (Object o : latest.attributes()) {
241
				final Attribute a = (Attribute) o;
242
				expired.addAttribute(a.getName(), a.getValue());
243
			}
244

  
245
			latest.addAttribute("id", rawSet.getId());
246
			latest.addAttribute("creationDate", rawSet.getCreationDate());
247
			latest.addAttribute("lastUpdate", rawSet.getLastUpdate());
248

  
249
			serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ActionManagerSetDSResourceType");
250
		} catch (Exception e) {
251
			log.error("Error updating profile of set: " + set);
252
			throw new ActionManagerException("Error running xquery: " + q, e);
253
		}
254
	}
255

  
256
	public void updateLastUpdate(final String rawSet) throws ActionManagerException {
257
		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType')//RAW_SETS/*[@id='" + rawSet
258
				+ "']/@lastUpdate return update replace $x with '" + DateUtils.now_ISO8601() + "'";
259

  
260
		try {
261
			serviceLocator.getService(ISRegistryService.class).executeXUpdate(q);
262
		} catch (Exception e) {
263
			log.error("Error updating lastUpdate using query: " + q, e);
264
			throw new ActionManagerException("Error updating lastUpdate using query: " + q, e);
265
		}
266
	}
267

  
268
	public StringTemplate getActionManagerSetDsTemplate() {
269
		return actionManagerSetDsTemplate;
270
	}
271

  
272
	public String getProfileId(final String set) throws ISLookUpException {
273
		final String xquery = "substring-before(for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
274
				+ "return $x[.//SET/@id = '%s']//RESOURCE_IDENTIFIER/@value/string(), '_')";
275
		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(String.format(xquery, set));
276
	}
277

  
278
	@Required
279
	public void setActionManagerSetDsTemplate(final StringTemplate actionManagerSetDsTemplate) {
280
		this.actionManagerSetDsTemplate = actionManagerSetDsTemplate;
281
	}
282

  
283
	public Endpoint getEndpoint() {
284
		return endpoint;
285
	}
286

  
287
	@Required
288
	public void setEndpoint(final Endpoint endpoint) {
289
		this.endpoint = endpoint;
290
	}
291

  
292
	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
293
		return eprBuilder;
294
	}
295

  
296
	@Required
297
	public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) {
298
		this.eprBuilder = eprBuilder;
299
	}
300

  
301
}
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/listSets.st
1
$inspector/actions/actionMaster(ait={
2

  
3
	<style type="text/css">
4
		#setList    { margin-top: 12px; }
5
		#setList td { padding: 4px; padding-left: 8px; padding-right: 8px; font-size: 11px;}
6
		#setList th { font-weight: bold; border-bottom: 1px solid #999; font-size: 14px; }
7
	</style>
8
	
9
	<br /><br />
10
	<form method="get" action="createSet.do">
11
		<b>New set:</b>
12
		ID: <input type="text" name="id" />
13
		Name: <input type="text" name="name" />
14
		Directory: <input type="text" name="directory" />
15
		<input type="submit" value="create">
16
	</form>
17
	<br />
18
	<table id="setList" cellspacing="0px">
19
		<thead>
20
			<tr>
21
				<th style="width: 100px">id</th>
22
				<th style="width: 240px">name</th>
23
				<th style="width: 120px">impact</th>
24
				<th style="width: 450px">latest raw set</th>
25
				<th style="width: 200px">latest raw set creation</th>
26
				<th style="width: 200px">latest raw set update</th>
27
				<th style="width: 100px">&nbsp;</th>
28
			</tr>
29
		</thead>
30
		<tbody>
31
			$sets:{s|
32
				<tr>
33
					<td><a href="linkActionSetProfile.do?set=$s.id$">$s.id$</a></td>
34
					<td>$s.name$</td>
35
					<td>$s.impact$</td>
36
					<td>$s.latest.id$</td>
37
					<td>$s.latest.creationDate$</td>
38
					<td>$s.latest.lastUpdate$</td>
39
					<td><a href="deleteSet.do?set=$s.id$">[delete]</a></td>
40
				</tr>
41
			}$	
42
		</tbody>
43
	</table>
44
	
45
})$
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/listAll.st
1
$inspector/actions/actionMaster(ait={
2
	<p>
3
		<b>All actions:</b>
4
	</p>
5

  
6
	<table id="actionList" cellspacing="0px">
7
		<tr class="header">
8
			<td>action ID</td>
9
			<td>set</td>		
10
			<td>date</td>
11
		</tr>
12
		$actions:{a|
13
		<tr>
14
			<td><b>$a.rowId$</b></td>
15
			<td>$a.set$</td>
16
			<td>$a.date$</td>
17
		</tr>
18
		}$
19
	</table>
20
	<br /><br />
21
	$if(next)$<a href="?type=$type$&start=$next$">[Continue >>]</a>$endif$
22

  
23
})$
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/commit.st
1
$inspector/actions/actionMaster(ait={
2
	<br />
3
	<form method="get">
4
		<b>Sets (No selection for ALL SETS):</b><br /><br />
5
		<select name="sets" multiple="multiple" size="10">
6
			$sets:{s|<option value="$s.id$">$s.name$</option>}$
7
		</select><br /><br />
8
		<input type="submit" value="promote">
9
	</form>
10
	<br />
11
	<hr />
12
	<br />
13
	$if(message)$
14
	<p>
15
		<b>$message$</b><br /><br />
16
		<a href="../msroAllProcesses.do">See orchestration processes</a><br /><br />
17
	</p>
18
	$endif$	
19
})$
20
	
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/garbage.st
1
$inspector/actions/actionMaster(ait={
2
	<br /><br />
3
	$if(message)$
4
		<p>
5
			<b>$message$</b><br /><br />
6
			<a href="../msroAllProcesses.do">See orchestration processes</a><br /><br />
7
		</p>
8
	$else$		
9
	
10
})$
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/enabling/views/inspector/actions/actionMaster.st
1
$inspector/master(it={
2
	<style type="text/css">
3
		#actionList { margin-top: 12px; }
4
		#actionList td { padding: 4px; padding-left: 8px; padding-right: 8px; font-size: 11px;}
5
		#actionList tr.header td { font-weight: bold; border-bottom: 1px solid #999; font-size: 14px; }
6
	</style>
7

  
8
	<h2>Action Manager service</h2>
9

  
10
	<p><b>Operation:</b>
11
		<a href="commit.do">promote actions</a> |
12
		<a href="garbage.do">garbage actions</a> |
13
		<a href="listSets.do">sets</a>
14
	</p>
15
	<hr />
16

  
17
	$ait$
18

  
19
})$
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/profiles/promoteActions.xml
1
<RESOURCE_PROFILE>
2
	<HEADER>
3
		<RESOURCE_IDENTIFIER
4
			value="8bb6c559-edf3-4da1-87d7-cdee4fba21dd_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ==" />
5
		<RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType" />
6
		<RESOURCE_KIND value="HadoopJobConfigurationDSResources" />
7
		<RESOURCE_URI value="" />
8
		<DATE_OF_CREATION value="2001-12-31T12:00:00" />
9
	</HEADER>
10
	<BODY>
11
		<HADOOP_JOB name="promoteActionsJob" type="mapreduce">
12
			<DESCRIPTION>map only job that promote actions from a set to the data table</DESCRIPTION>
13
			<STATIC_CONFIGURATION>
14
				<PROPERTY key="mapreduce.map.class"	value="eu.dnetlib.data.mapreduce.hbase.actions2.PromoteActionsMapper" />
15
				<PROPERTY key="mapreduce.inputformat.class"	value="org.apache.hadoop.hbase.mapreduce.TableInputFormat" />
16
				<PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableOutputFormat" />
17
				<PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text" />
18
				<PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.hbase.client.Put" />
19
				<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" />
20
				<PROPERTY key="mapreduce.map.speculative" value="false" />
21
				<PROPERTY key="mapred.reduce.tasks" value="0" />
22
				<!-- <PROPERTY key="job.lib" value="/tmp/dnet-mapreduce-jobs-0.0.3-SNAPSHOT-jar-with-dependencies.jar" />  -->
23
			</STATIC_CONFIGURATION>
24

  
25
			<JOB_INTERFACE>
26
				<PARAM name="hbase.mapred.inputtable" required="true" description="source hbase table" />
27
				<PARAM name="hbase.mapred.outputtable" required="true" description="target hbase table" />
28
				<PARAM name="hbase.mapreduce.inputtable" required="true" description="source hbase table" />
29
				<PARAM name="hbase.mapreduce.outputtable" required="true" description="target hbase table" />
30
				<PARAM name="latestRawSets" required="true" description="the latest action raw sets" />
31
			</JOB_INTERFACE>
32

  
33
			<SCAN>
34
				<FILTERS operator="MUST_PASS_ALL">
35
					<FILTER type="prefix" value="aac|" />
36
				</FILTERS>
37
				<FAMILIES />
38
			</SCAN>
39
			
40
		</HADOOP_JOB>
41
		
42
		<STATUS>
43
			<LAST_SUBMISSION_DATE value="2001-12-31T12:00:00" />
44
			<RUNNING_INSTANCES value="0" />
45
			<CUMULATIVE_RUN value="0" />
46
		</STATUS>
47
		
48
		<SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
49
	</BODY>
50
</RESOURCE_PROFILE>
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/profiles/garbageActions.xml
1
<RESOURCE_PROFILE>
2
	<HEADER>
3
		<RESOURCE_IDENTIFIER
4
			value="58c652e5-4789-47fe-87f2-c511ba6f72ea_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ==" />
5
		<RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType" />
6
		<RESOURCE_KIND value="HadoopJobConfigurationDSResources" />
7
		<RESOURCE_URI value="" />
8
		<DATE_OF_CREATION value="2001-12-31T12:00:00" />
9
	</HEADER>
10
	<BODY>
11
		<HADOOP_JOB name="garbageActionsJob" type="mapreduce">
12
			<DESCRIPTION>map only job that delete actions from a set
13
			</DESCRIPTION>
14
			<STATIC_CONFIGURATION>
15
				<PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.actions2.GarbageActionsMapper" />
16
				<PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat" />
17
				<PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableOutputFormat" />
18
				<PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text" />
19
				<PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.hbase.client.Delete" />
20
				<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" />
21
				<PROPERTY key="mapreduce.map.speculative" value="false" />
22
				<PROPERTY key="mapred.reduce.tasks" value="0" />
23
				<!-- <PROPERTY key="job.lib" value="/tmp/dnet-mapreduce-jobs-0.0.3-SNAPSHOT-jar-with-dependencies.jar" />  -->
24
			</STATIC_CONFIGURATION>
25
			<JOB_INTERFACE>
26
				<PARAM name="hbase.mapred.inputtable" required="true" description="source hbase table" />
27
				<PARAM name="hbase.mapred.outputtable" required="true" description="target hbase table" />
28
				<PARAM name="hbase.mapreduce.inputtable" required="true" description="source hbase table" />
29
				<PARAM name="hbase.mapreduce.outputtable" required="true" description="target hbase table" />
30
				<PARAM name="latestRawSets" required="true" description="the latest action raw sets" />
31
			</JOB_INTERFACE>
32
        	<SCAN>
33
        		<FILTERS/>
34
        		<FAMILIES>
35
        			<FAMILY value="set" />
36
        		</FAMILIES>
37
        	</SCAN>
38
		</HADOOP_JOB>
39
		<STATUS>
40
			<LAST_SUBMISSION_DATE value="2001-12-31T12:00:00" />
41
			<RUNNING_INSTANCES value="0" />
42
			<CUMULATIVE_RUN value="0" />
43
		</STATUS>
44
		
45
		<SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
46
	</BODY>
47
</RESOURCE_PROFILE>
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/test/schemas/ActionManagerSetDSResourceType.xsd
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
3
	elementFormDefault="qualified">
4
	<xs:element name="RESOURCE_PROFILE">
5
		<xs:complexType>
6
			<xs:sequence>
7
				<xs:element name="HEADER">
8
					<xs:complexType>
9
						<xs:sequence>
10
							<xs:element name="RESOURCE_IDENTIFIER">
11
								<xs:complexType>
12
									<xs:attribute name="value" use="required" />
13
								</xs:complexType>
14
							</xs:element>
15
							<xs:element name="RESOURCE_TYPE">
16
								<xs:complexType>
17
									<xs:attribute name="value" use="required" type="xs:NCName" />
18
								</xs:complexType>
19
							</xs:element>
20
							<xs:element name="RESOURCE_KIND">
21
								<xs:complexType>
22
									<xs:attribute name="value" use="required" type="xs:NCName" />
23
								</xs:complexType>
24
							</xs:element>
25
							<xs:element name="RESOURCE_URI">
26
								<xs:complexType>
27
									<xs:attribute name="value" use="required" type="xs:anyURI" />
28
								</xs:complexType>
29
							</xs:element>
30
							<xs:element name="DATE_OF_CREATION">
31
								<xs:complexType>
32
									<xs:attribute name="value" use="required" />
33
								</xs:complexType>
34
							</xs:element>
35
						</xs:sequence>
36
					</xs:complexType>
37
				</xs:element>
38
				<xs:element name="BODY">
39
					<xs:complexType>
40
						<xs:sequence>
41
							<xs:element name="SET">
42
								<xs:complexType>
43
									<xs:simpleContent>
44
										<xs:extension base="xs:string">
45
											<xs:attribute name="id" use="required" type="xs:string" />
46
											<xs:attribute name="directory" use="optional" type="xs:string"/>
47
										</xs:extension>
48
									</xs:simpleContent>
49
								</xs:complexType>
50
							</xs:element>
51
							<xs:element name="IMPACT" type="xs:NCName" />
52
							<xs:element name="RAW_SETS">
53
								<xs:complexType>
54
									<xs:sequence>
55
										<xs:element name="LATEST" minOccurs="0" maxOccurs="1"
56
											type="RAW_SET_TYPE" />
57
										<xs:element name="EXPIRED" minOccurs="0" maxOccurs="unbounded"
58
											type="RAW_SET_TYPE" />
59
									</xs:sequence>
60
								</xs:complexType>
61
							</xs:element>
62

  
63
							<xs:element name="STATUS">
64
								<xs:complexType>
65
									<xs:sequence>
66
										<xs:element name="LAST_EXECUTION_DATE" type="xs:string"
67
											nillable="true" />
68
										<xs:element name="NUMBER_OF_INFOPACKAGES" type="xs:integer" />
69
										<xs:element name="NUMBER_OF_ATOMICACTIONS" type="xs:integer" />
70
									</xs:sequence>
71
								</xs:complexType>
72
							</xs:element>
73
						</xs:sequence>
74
					</xs:complexType>
75
				</xs:element>
76
			</xs:sequence>
77
		</xs:complexType>
78
	</xs:element>
79

  
80
	<xs:complexType name="RAW_SET_TYPE">
81
		<xs:attribute name="id" type="xs:string" use="required" />
82
		<xs:attribute name="creationDate" type="xs:string" use="required" />
83
		<xs:attribute name="lastUpdate" type="xs:string" use="required" />
84
	</xs:complexType>
85

  
86
</xs:schema>
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/actionmanager/applicationContext-dnet-actionmanager-service.properties
1
hbase.actions.table=db_actions
2
services.actionmanager.garbage.time.maring=345600000
3
services.actionmanager.hbase.client.startdelay=60000
4
services.actionmanager.basePath=/openaire/actionManager
5
services.actionmanager.garbage.retain.threshold=3
6
services.actionmanager.promote.mapper.class=eu.dnetlib.data.mapreduce.hbase.actions2.PromoteActionSetFromHDFSMapper
7

  
modules/dnet-actionmanager-service/tags/dnet-actionmanager-service-5.2.1/src/main/resources/eu/dnetlib/actionmanager/applicationContext-dnet-actionmanager-service.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3
       xmlns:jaxws="http://cxf.apache.org/jaxws" xmlns:sec="http://cxf.apache.org/configuration/security"
4
       xmlns:wsa="http://cxf.apache.org/ws/addressing" xmlns:p="http://www.springframework.org/schema/p"
5
       xmlns:http="http://cxf.apache.org/transports/http/configuration" xmlns:t="http://dnetlib.eu/springbeans/t"
6
       xmlns:template="http://dnetlib.eu/springbeans/template" xmlns:util="http://www.springframework.org/schema/util"
7
       xmlns="http://www.springframework.org/schema/beans"
8
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
9
                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
10
                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
11
                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
12
                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
13
                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd
14
                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
15

  
16
	<bean id="actionManagerService" class="eu.dnetlib.actionmanager.ActionManagerServiceImpl"
17
		init-method="start" p:notificationHandler-ref="actionManagerNotificationHandler"
18
		destroy-method="stop" />
19
		
20
	<bean
21
		id="actionManagerNotificationHandler"
22
		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"
23
		p:blackboardExecutor-ref="actionManagerBlackboardExecutor" />
24
	
25
	<bean id="actionManagerBlackboardExecutor"
26
		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
27
		p:blackboardHandler-ref="blackboardHandler"
28
		p:actionType="eu.dnetlib.actionmanager.blackboard.ActionManagerActions"
29
		p:incomplete="false">
30
		<property name="actionMap">
31
			<map>
32
				<entry key="GARBAGE_HDFS">
33
					<bean class="eu.dnetlib.actionmanager.blackboard.GarbageActionManagerActionFromHDFS"/>
34
				</entry>
35
				<entry key="PROMOTE_FROM_HDFS">
36
					<bean class="eu.dnetlib.actionmanager.blackboard.PromoteActionManagerActionFromHDFS"/>
37
				</entry>
38

  
39
			</map>
40
		</property>
41
		
42
	</bean>
43

  
44
	<!-- M/R Job launcher -->
45
	<bean id="actionmanagerHbaseJobLauncher" class="eu.dnetlib.actionmanager.blackboard.JobLauncher"
46
		p:actionTable="${hbase.actions.table}"
47
		p:dataTable="${hbase.mapred.datatable}" 
48
		p:infomationServiceClient-ref="actionmanagerISClient"/>
49
		
50
	<!-- endpoints -->
51
	<jaxws:endpoint id="actionManagerServiceEndpoint"
52
		implementor="#actionManagerService" implementorClass="eu.dnetlib.actionmanager.rmi.ActionManagerService"
53
		address="/actionManager" />
54
	
55
	<template:instance name="serviceRegistrationManager"
56
		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"
57
		t:name="actionManagerServiceRegistrationManager" t:service="actionManagerService"
58
		t:endpoint="actionManagerServiceEndpoint" t:jobScheduler="jobScheduler"
59
		t:serviceRegistrator="actionmanagerServiceRegistrator" />
60
		
61
	<bean id="actionmanagerServiceRegistrator"
62
		class="eu.dnetlib.enabling.tools.registration.BlackboardServiceRegistrator"
63
		p:serviceLocator-ref="uniqueServiceLocator"
64
		p:serviceNameResolver-ref="defaultServiceNameResolver"
65
		p:hnmLocator-ref="${services.registration.default.hnmlocator}"
66
		p:eprBuilder-ref="jaxwsEndpointReferenceBuilder">
67
		<property name="serviceProperties">
68
			<map>
69
				<entry key="garbageRetainThreshold" value="${services.actionmanager.garbage.retain.threshold}"/>
70
				<entry key="garbageTimeMargin" value="${services.actionmanager.garbage.time.maring}" />
71
				<entry key="basePath" value="${services.actionmanager.basePath}" />
72
			</map>
73
		</property>
74
	</bean>				
75

  
76
	<!-- Information Service Client -->
77
	<bean id="actionmanagerISClient" class="eu.dnetlib.actionmanager.is.ISClient"
78
		p:actionManagerSetDsTemplate-ref="actionManagerSetDsTemplate"
79
		p:endpoint-ref="actionManagerServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />
80

  
81
	<bean id="actionManagerSetDsTemplate"
82
		class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
83
		p:template="classpath:/eu/dnetlib/actionmanager/tmpl/actionManagerSetDS.xml.tmpl"
84
		scope="prototype" />
85

  
86

  
87
</beans>
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff