Revision 45292
Added by Claudio Atzori almost 7 years ago
modules/dnet-hadoop-service-rmi/trunk/deploy.info | ||
---|---|---|
1 |
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-hadoop-service-rmi/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-hadoop-service-rmi"} |
modules/dnet-hadoop-service-rmi/trunk/src/test/java/eu/dnetlib/data/hadoop/rmi/hbase/schema/HBaseTableDescriptorTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase.schema; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.io.InputStream; |
|
5 |
|
|
6 |
import org.apache.commons.io.IOUtils; |
|
7 |
import org.junit.Test; |
|
8 |
|
|
9 |
import static org.junit.Assert.assertFalse; |
|
10 |
import static org.junit.Assert.assertNotNull; |
|
11 |
|
|
12 |
/** |
|
13 |
* Created by claudio on 11/11/15. |
|
14 |
*/ |
|
15 |
public class HBaseTableDescriptorTest { |
|
16 |
|
|
17 |
@Test |
|
18 |
public void testSerialise() throws IOException { |
|
19 |
|
|
20 |
final InputStream is = getClass().getResourceAsStream("hbaseTable.json"); |
|
21 |
|
|
22 |
final String json = IOUtils.toString(is); |
|
23 |
|
|
24 |
final HBaseTableDescriptor desc = HBaseTableDescriptor.fromJSON(json); |
|
25 |
|
|
26 |
assertNotNull(desc); |
|
27 |
assertNotNull(desc.getColumns()); |
|
28 |
assertNotNull(desc.getRegions()); |
|
29 |
|
|
30 |
assertFalse(desc.getColumns().isEmpty()); |
|
31 |
assertFalse(desc.getRegions().isEmpty()); |
|
32 |
|
|
33 |
System.out.println(desc.toString()); |
|
34 |
} |
|
35 |
|
|
36 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/test/java/eu/dnetlib/data/hadoop/rmi/hbase/HBaseRowDescriptorTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
|
|
5 |
import com.google.common.collect.Lists; |
|
6 |
import org.junit.Test; |
|
7 |
|
|
8 |
import static org.junit.Assert.assertEquals; |
|
9 |
|
|
10 |
/** |
|
11 |
* Created by claudio on 23/10/15. |
|
12 |
*/ |
|
13 |
public class HBaseRowDescriptorTest { |
|
14 |
|
|
15 |
@Test |
|
16 |
public void testSerialisation() { |
|
17 |
|
|
18 |
HBaseRowDescriptor d1 = new HBaseRowDescriptor(); |
|
19 |
|
|
20 |
d1.setRowKey("k1"); |
|
21 |
d1.getColumns().add(new Column("c1", l("q1", "q2"))); |
|
22 |
d1.getColumns().add(new Column("c2", l("q1", "q2"))); |
|
23 |
|
|
24 |
final String d1s = d1.toString(); |
|
25 |
System.out.println(d1s); |
|
26 |
|
|
27 |
|
|
28 |
final HBaseRowDescriptor d2 = HBaseRowDescriptor.fromJSON(d1s); |
|
29 |
|
|
30 |
assertEquals(d1.toString(), d2.toString()); |
|
31 |
} |
|
32 |
|
|
33 |
private List<String> l(String... s) { |
|
34 |
return Lists.newArrayList(s); |
|
35 |
} |
|
36 |
|
|
37 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/test/resources/eu/dnetlib/data/hadoop/rmi/hbase/schema/hbaseTable.json | ||
---|---|---|
1 |
{ |
|
2 |
"columns": [ |
|
3 |
"result", |
|
4 |
"resultProject_outcome_isProducedBy", |
|
5 |
"person", |
|
6 |
"resultResult_similarity_hasAmongTopNSimilarDocuments", |
|
7 |
"resultResult_dedup_merges", |
|
8 |
"organizationOrganization_dedup_merges", |
|
9 |
"projectPerson_contactPerson_isContact", |
|
10 |
"personPerson_dedup_merges", |
|
11 |
"personPerson_coauthorship_isCoauthorOf", |
|
12 |
"personResult_authorship_hasAuthor", |
|
13 |
"organization", |
|
14 |
"resultProject_outcome_produces", |
|
15 |
"resultResult_similarity_isAmongTopNSimilarDocuments", |
|
16 |
"datasourceOrganization_provision_provides", |
|
17 |
"resultOrganization_affiliation_isAffiliatedWith", |
|
18 |
"organizationOrganization_dedupSimilarity_isSimilarTo", |
|
19 |
"datasource", |
|
20 |
"personPerson_dedupSimilarity_isSimilarTo", |
|
21 |
"resultResult_publicationDataset_isRelatedTo", |
|
22 |
"organizationOrganization_dedup_isMergedIn", |
|
23 |
"projectOrganization_participation_hasParticipant", |
|
24 |
"projectOrganization_participation_isParticipant", |
|
25 |
"project", |
|
26 |
"personResult_authorship_isAuthorOf", |
|
27 |
"resultResult_dedup_isMergedIn", |
|
28 |
"personPerson_dedup_isMergedIn", |
|
29 |
"projectPerson_contactPerson_hasContact", |
|
30 |
"datasourceOrganization_provision_isProvidedBy", |
|
31 |
"resultResult_dedupSimilarity_isSimilarTo" |
|
32 |
], |
|
33 |
"regions": [ |
|
34 |
{ |
|
35 |
"startKey": "", |
|
36 |
"endKey": "30|dedup_wf_001::01f6ec93ba93d5dee29436e446a19cff" |
|
37 |
}, |
|
38 |
{ |
|
39 |
"startKey": "30|dedup_wf_001::01f6ec93ba93d5dee29436e446a19cff", |
|
40 |
"endKey": "30|dedup_wf_001::06cdeeae3ffe195d2d346c8f969d3733" |
|
41 |
}, |
|
42 |
{ |
|
43 |
"startKey": "30|dedup_wf_001::06cdeeae3ffe195d2d346c8f969d3733", |
|
44 |
"endKey": "30|dedup_wf_001::0f9e3af0a62c397f5c79b3040e4fc2cb" |
|
45 |
}, |
|
46 |
{ |
|
47 |
"startKey": "30|dedup_wf_001::0f9e3af0a62c397f5c79b3040e4fc2cb", |
|
48 |
"endKey": "30|dedup_wf_001::18037d1fc2e9f4766074b31ba355622c" |
|
49 |
}, |
|
50 |
{ |
|
51 |
"startKey": "30|dedup_wf_001::18037d1fc2e9f4766074b31ba355622c", |
|
52 |
"endKey": "30|dedup_wf_001::1fb797d8e394bb01ac35a662dde2b0f8" |
|
53 |
}, |
|
54 |
{ |
|
55 |
"startKey": "30|dedup_wf_001::1fb797d8e394bb01ac35a662dde2b0f8", |
|
56 |
"endKey": "30|dedup_wf_001::268b0ae37189c8dadb8f13ab0606c97f" |
|
57 |
}, |
|
58 |
{ |
|
59 |
"startKey": "30|dedup_wf_001::268b0ae37189c8dadb8f13ab0606c97f", |
|
60 |
"endKey": "30|dedup_wf_001::30ca5b71cafc2f5f1b9e863469e532d5" |
|
61 |
}, |
|
62 |
{ |
|
63 |
"startKey": "30|dedup_wf_001::30ca5b71cafc2f5f1b9e863469e532d5", |
|
64 |
"endKey": "30|dedup_wf_001::3c06a9d133ac0a6bc811eec8328c977e" |
|
65 |
}, |
|
66 |
{ |
|
67 |
"startKey": "30|dedup_wf_001::3c06a9d133ac0a6bc811eec8328c977e", |
|
68 |
"endKey": "30|dedup_wf_001::44befcc568dbe380106b7f6fe9ef8307" |
|
69 |
}, |
|
70 |
{ |
|
71 |
"startKey": "30|dedup_wf_001::44befcc568dbe380106b7f6fe9ef8307", |
|
72 |
"endKey": "30|dedup_wf_001::4da577deebb7d8b2c18874b1e715a482" |
|
73 |
}, |
|
74 |
{ |
|
75 |
"startKey": "30|dedup_wf_001::4da577deebb7d8b2c18874b1e715a482", |
|
76 |
"endKey": "30|dedup_wf_001::563c55f0830ce083e4c410b1f4f59c55" |
|
77 |
}, |
|
78 |
{ |
|
79 |
"startKey": "30|dedup_wf_001::563c55f0830ce083e4c410b1f4f59c55", |
|
80 |
"endKey": "30|dedup_wf_001::636f3edbe11558886c4ba24c5c09ea89" |
|
81 |
}, |
|
82 |
{ |
|
83 |
"startKey": "30|dedup_wf_001::636f3edbe11558886c4ba24c5c09ea89", |
|
84 |
"endKey": "30|dedup_wf_001::6dd9d6802123c6063b4ce706a8748624" |
|
85 |
}, |
|
86 |
{ |
|
87 |
"startKey": "30|dedup_wf_001::6dd9d6802123c6063b4ce706a8748624", |
|
88 |
"endKey": "30|dedup_wf_001::76b663bd033d59e1b17f37cc8b0a80d8" |
|
89 |
}, |
|
90 |
{ |
|
91 |
"startKey": "30|dedup_wf_001::76b663bd033d59e1b17f37cc8b0a80d8", |
|
92 |
"endKey": "30|dedup_wf_001::7f056a7b3d7410087f317d6630c526d3" |
|
93 |
}, |
|
94 |
{ |
|
95 |
"startKey": "30|dedup_wf_001::7f056a7b3d7410087f317d6630c526d3", |
|
96 |
"endKey": "30|dedup_wf_001::8b80a91fc8c4045ceb650a22b495c18e" |
|
97 |
}, |
|
98 |
{ |
|
99 |
"startKey": "30|dedup_wf_001::8b80a91fc8c4045ceb650a22b495c18e", |
|
100 |
"endKey": "30|dedup_wf_001::964cad44b7c717bc9b6166a329b2c19f" |
|
101 |
}, |
|
102 |
{ |
|
103 |
"startKey": "30|dedup_wf_001::964cad44b7c717bc9b6166a329b2c19f", |
|
104 |
"endKey": "30|dedup_wf_001::a05a81f85cbd08f4ef8ea424ab1f6c2d" |
|
105 |
}, |
|
106 |
{ |
|
107 |
"startKey": "30|dedup_wf_001::a05a81f85cbd08f4ef8ea424ab1f6c2d", |
|
108 |
"endKey": "30|dedup_wf_001::aa24cb2a368b4775b39c0d2d524959d3" |
|
109 |
}, |
|
110 |
{ |
|
111 |
"startKey": "30|dedup_wf_001::aa24cb2a368b4775b39c0d2d524959d3", |
|
112 |
"endKey": "30|dedup_wf_001::b99149295238e372c7a41f30480600a7" |
|
113 |
}, |
|
114 |
{ |
|
115 |
"startKey": "30|dedup_wf_001::b99149295238e372c7a41f30480600a7", |
|
116 |
"endKey": "30|dedup_wf_001::c6e679f5bade940987bf3ab69e76cf8c" |
|
117 |
}, |
|
118 |
{ |
|
119 |
"startKey": "30|dedup_wf_001::c6e679f5bade940987bf3ab69e76cf8c", |
|
120 |
"endKey": "30|dedup_wf_001::d377cfadcd3799b5c509addc4a99c3bb" |
|
121 |
}, |
|
122 |
{ |
|
123 |
"startKey": "30|dedup_wf_001::d377cfadcd3799b5c509addc4a99c3bb", |
|
124 |
"endKey": "30|dedup_wf_001::e10b99fb6d1f8698077ca9b4cfb701b9" |
|
125 |
}, |
|
126 |
{ |
|
127 |
"startKey": "30|dedup_wf_001::e10b99fb6d1f8698077ca9b4cfb701b9", |
|
128 |
"endKey": "30|dedup_wf_001::ed0fff7f00ee9b2101836c95e9e09d23" |
|
129 |
}, |
|
130 |
{ |
|
131 |
"startKey": "30|dedup_wf_001::ed0fff7f00ee9b2101836c95e9e09d23", |
|
132 |
"endKey": "30|dedup_wf_001::f7a73e3a03a09a10ee7f7697122f0681" |
|
133 |
}, |
|
134 |
{ |
|
135 |
"startKey": "30|dedup_wf_001::f7a73e3a03a09a10ee7f7697122f0681", |
|
136 |
"endKey": "30|doajarticles::0fdd94fa342532f903b50e88a99e10bd" |
|
137 |
}, |
|
138 |
{ |
|
139 |
"startKey": "30|doajarticles::0fdd94fa342532f903b50e88a99e10bd", |
|
140 |
"endKey": "30|doajarticles::451d3a1643b5be43f4ae2a34cdd27937" |
|
141 |
}, |
|
142 |
{ |
|
143 |
"startKey": "30|doajarticles::451d3a1643b5be43f4ae2a34cdd27937", |
|
144 |
"endKey": "30|doajarticles::71a1fc746b5365f574e3156747f18745" |
|
145 |
}, |
|
146 |
{ |
|
147 |
"startKey": "30|doajarticles::71a1fc746b5365f574e3156747f18745", |
|
148 |
"endKey": "30|doajarticles::98b1fe3e237757b6eb4de63baafd1923" |
|
149 |
}, |
|
150 |
{ |
|
151 |
"startKey": "30|doajarticles::98b1fe3e237757b6eb4de63baafd1923", |
|
152 |
"endKey": "30|doajarticles::c67bbcd47b4be3ac87dd57dee962f75b" |
|
153 |
}, |
|
154 |
{ |
|
155 |
"startKey": "30|doajarticles::c67bbcd47b4be3ac87dd57dee962f75b", |
|
156 |
"endKey": "30|doajarticles::f34078dfbfb6474825db277fa83b393b" |
|
157 |
}, |
|
158 |
{ |
|
159 |
"startKey": "30|doajarticles::f34078dfbfb6474825db277fa83b393b", |
|
160 |
"endKey": "30|frontiers___::977250f5038a72ca8903dd0caa49d375" |
|
161 |
}, |
|
162 |
{ |
|
163 |
"startKey": "30|frontiers___::977250f5038a72ca8903dd0caa49d375", |
|
164 |
"endKey": "30|od______1064::1892eb3502552398aafbcc80ec822556" |
|
165 |
}, |
|
166 |
{ |
|
167 |
"startKey": "30|od______1064::1892eb3502552398aafbcc80ec822556", |
|
168 |
"endKey": "30|od______1064::4c7dc52ff8927d3a4d3097ddfa6affa0" |
|
169 |
}, |
|
170 |
{ |
|
171 |
"startKey": "30|od______1064::4c7dc52ff8927d3a4d3097ddfa6affa0", |
|
172 |
"endKey": "30|od______1064::77f5c1d6ee91c1e278148a135315a0b2" |
|
173 |
}, |
|
174 |
{ |
|
175 |
"startKey": "30|od______1064::77f5c1d6ee91c1e278148a135315a0b2", |
|
176 |
"endKey": "30|od______1064::a50d37c3bf2951b531c7fb1911c4fe5b" |
|
177 |
}, |
|
178 |
{ |
|
179 |
"startKey": "30|od______1064::a50d37c3bf2951b531c7fb1911c4fe5b", |
|
180 |
"endKey": "30|od______1064::d281738255d0caf028e142d83bb23194" |
|
181 |
}, |
|
182 |
{ |
|
183 |
"startKey": "30|od______1064::d281738255d0caf028e142d83bb23194", |
|
184 |
"endKey": "30|od______1106::41f8c3d8d5acef5d0c00fe7c187ca808" |
|
185 |
}, |
|
186 |
{ |
|
187 |
"startKey": "30|od______1106::41f8c3d8d5acef5d0c00fe7c187ca808", |
|
188 |
"endKey": "30|od______1222::32ef3ea6510baca180965a9ee42cdfb1" |
|
189 |
}, |
|
190 |
{ |
|
191 |
"startKey": "30|od______1222::32ef3ea6510baca180965a9ee42cdfb1", |
|
192 |
"endKey": "30|od______1412::25f6b55fd946f8b2a81747a8c5202dc7" |
|
193 |
}, |
|
194 |
{ |
|
195 |
"startKey": "30|od______1412::25f6b55fd946f8b2a81747a8c5202dc7", |
|
196 |
"endKey": "30|od______2001::246e27b8c42ee8e0580958eedc79e28b" |
|
197 |
}, |
|
198 |
{ |
|
199 |
"startKey": "30|od______2001::246e27b8c42ee8e0580958eedc79e28b", |
|
200 |
"endKey": "30|od______3061::fae9705d71631c06b9e6b36d14f81661" |
|
201 |
}, |
|
202 |
{ |
|
203 |
"startKey": "30|od______3061::fae9705d71631c06b9e6b36d14f81661", |
|
204 |
"endKey": "30|od_______165::194583341339bd02e9e02422926bf630" |
|
205 |
}, |
|
206 |
{ |
|
207 |
"startKey": "30|od_______165::194583341339bd02e9e02422926bf630", |
|
208 |
"endKey": "30|od_______165::9b47f3c8f28ce97d32350389104f9899" |
|
209 |
}, |
|
210 |
{ |
|
211 |
"startKey": "30|od_______165::9b47f3c8f28ce97d32350389104f9899", |
|
212 |
"endKey": "30|od_______185::453155023cbb2afeca98b640d6594a08" |
|
213 |
}, |
|
214 |
{ |
|
215 |
"startKey": "30|od_______185::453155023cbb2afeca98b640d6594a08", |
|
216 |
"endKey": "30|od_______322::84120a66fff81cd34cbc8cbef3edd715" |
|
217 |
}, |
|
218 |
{ |
|
219 |
"startKey": "30|od_______322::84120a66fff81cd34cbc8cbef3edd715", |
|
220 |
"endKey": "30|od_______484::2a821095a78aa2b2ad82d73b2c5ad62a" |
|
221 |
}, |
|
222 |
{ |
|
223 |
"startKey": "30|od_______484::2a821095a78aa2b2ad82d73b2c5ad62a", |
|
224 |
"endKey": "30|od_______603::0b7b6c2731fb1fc372ef59e1cc2faf63" |
|
225 |
}, |
|
226 |
{ |
|
227 |
"startKey": "30|od_______603::0b7b6c2731fb1fc372ef59e1cc2faf63", |
|
228 |
"endKey": "30|od_______908::09d20b1f39dfc14eb45acd5dda8d4a21" |
|
229 |
}, |
|
230 |
{ |
|
231 |
"startKey": "30|od_______908::09d20b1f39dfc14eb45acd5dda8d4a21", |
|
232 |
"endKey": "30|od_______908::2087f4d03e16fae94512220255a3cd1d" |
|
233 |
}, |
|
234 |
{ |
|
235 |
"startKey": "30|od_______908::2087f4d03e16fae94512220255a3cd1d", |
|
236 |
"endKey": "30|od_______908::31abbab96ceda6e60d5dccd140f58e3c" |
|
237 |
}, |
|
238 |
{ |
|
239 |
"startKey": "30|od_______908::31abbab96ceda6e60d5dccd140f58e3c", |
|
240 |
"endKey": "30|od_______908::45ddf56ad9ee5360bf1a1a191b3baeb8" |
|
241 |
}, |
|
242 |
{ |
|
243 |
"startKey": "30|od_______908::45ddf56ad9ee5360bf1a1a191b3baeb8", |
|
244 |
"endKey": "30|od_______908::5b6b219395b454fb0bd8a394c5070912" |
|
245 |
}, |
|
246 |
{ |
|
247 |
"startKey": "30|od_______908::5b6b219395b454fb0bd8a394c5070912", |
|
248 |
"endKey": "30|od_______908::6bd349faba19949050732b44b6269abb" |
|
249 |
}, |
|
250 |
{ |
|
251 |
"startKey": "30|od_______908::6bd349faba19949050732b44b6269abb", |
|
252 |
"endKey": "30|od_______908::7f53ececb9bb91daec9e2a9eaa7bbc4c" |
|
253 |
}, |
|
254 |
{ |
|
255 |
"startKey": "30|od_______908::7f53ececb9bb91daec9e2a9eaa7bbc4c", |
|
256 |
"endKey": "30|od_______908::8ed6cd6e2bbdeb9025948accab37b26f" |
|
257 |
}, |
|
258 |
{ |
|
259 |
"startKey": "30|od_______908::8ed6cd6e2bbdeb9025948accab37b26f", |
|
260 |
"endKey": "30|od_______908::a1d2f2e7212b3f7bbe6cdabc668c7769" |
|
261 |
}, |
|
262 |
{ |
|
263 |
"startKey": "30|od_______908::a1d2f2e7212b3f7bbe6cdabc668c7769", |
|
264 |
"endKey": "30|od_______908::b683f5a3b98b03154b8a5150d036eaba" |
|
265 |
}, |
|
266 |
{ |
|
267 |
"startKey": "30|od_______908::b683f5a3b98b03154b8a5150d036eaba", |
|
268 |
"endKey": "30|od_______908::cc0e85209e37d16b0be196dc3d173c8a" |
|
269 |
}, |
|
270 |
{ |
|
271 |
"startKey": "30|od_______908::cc0e85209e37d16b0be196dc3d173c8a", |
|
272 |
"endKey": "30|od_______908::dd2df143167b3b9040eb58436a22378d" |
|
273 |
}, |
|
274 |
{ |
|
275 |
"startKey": "30|od_______908::dd2df143167b3b9040eb58436a22378d", |
|
276 |
"endKey": "30|od_______908::f10a377825e517763a631a8e3cae6468" |
|
277 |
}, |
|
278 |
{ |
|
279 |
"startKey": "30|od_______908::f10a377825e517763a631a8e3cae6468", |
|
280 |
"endKey": "30|od_______961::b2172ece477c3b2bb379c41a70f2a729" |
|
281 |
}, |
|
282 |
{ |
|
283 |
"startKey": "30|od_______961::b2172ece477c3b2bb379c41a70f2a729", |
|
284 |
"endKey": "30|od________18::25b7030f6520a8c34f9c0f9c1bafb001" |
|
285 |
}, |
|
286 |
{ |
|
287 |
"startKey": "30|od________18::25b7030f6520a8c34f9c0f9c1bafb001", |
|
288 |
"endKey": "30|od________18::70f2be65de065ff0ae060b9fcf431c1d" |
|
289 |
}, |
|
290 |
{ |
|
291 |
"startKey": "30|od________18::70f2be65de065ff0ae060b9fcf431c1d", |
|
292 |
"endKey": "30|od________18::bf3e05da5f0bf7d0931e0b6f30924b33" |
|
293 |
}, |
|
294 |
{ |
|
295 |
"startKey": "30|od________18::bf3e05da5f0bf7d0931e0b6f30924b33", |
|
296 |
"endKey": "50|dedup_wf_001::02034127aa0e85feaa5f73e673c7c0d7" |
|
297 |
}, |
|
298 |
{ |
|
299 |
"startKey": "50|dedup_wf_001::02034127aa0e85feaa5f73e673c7c0d7", |
|
300 |
"endKey": "50|dedup_wf_001::0c6d39819631cb0049875aaa0e3f1fdc" |
|
301 |
}, |
|
302 |
{ |
|
303 |
"startKey": "50|dedup_wf_001::0c6d39819631cb0049875aaa0e3f1fdc", |
|
304 |
"endKey": "50|dedup_wf_001::16c764873dd9cd30039cfa6eafb66633" |
|
305 |
}, |
|
306 |
{ |
|
307 |
"startKey": "50|dedup_wf_001::16c764873dd9cd30039cfa6eafb66633", |
|
308 |
"endKey": "50|dedup_wf_001::2256f1d993f2e2dc3ee261a3221e9898" |
|
309 |
}, |
|
310 |
{ |
|
311 |
"startKey": "50|dedup_wf_001::2256f1d993f2e2dc3ee261a3221e9898", |
|
312 |
"endKey": "50|dedup_wf_001::2f20fc3902b09c13cb611556f88b6051" |
|
313 |
}, |
|
314 |
{ |
|
315 |
"startKey": "50|dedup_wf_001::2f20fc3902b09c13cb611556f88b6051", |
|
316 |
"endKey": "50|dedup_wf_001::3d120775a140a00e3265ebde52b47118" |
|
317 |
}, |
|
318 |
{ |
|
319 |
"startKey": "50|dedup_wf_001::3d120775a140a00e3265ebde52b47118", |
|
320 |
"endKey": "50|dedup_wf_001::4bc418b1fe54e4fa2a05629a06dd3a4c" |
|
321 |
}, |
|
322 |
{ |
|
323 |
"startKey": "50|dedup_wf_001::4bc418b1fe54e4fa2a05629a06dd3a4c", |
|
324 |
"endKey": "50|dedup_wf_001::5c6a6cce61901445bef19011e7a9cdd3" |
|
325 |
}, |
|
326 |
{ |
|
327 |
"startKey": "50|dedup_wf_001::5c6a6cce61901445bef19011e7a9cdd3", |
|
328 |
"endKey": "50|dedup_wf_001::6cccee8a482edd160d8b32500d210451" |
|
329 |
}, |
|
330 |
{ |
|
331 |
"startKey": "50|dedup_wf_001::6cccee8a482edd160d8b32500d210451", |
|
332 |
"endKey": "50|dedup_wf_001::7f03900f6f068d95d144cca696eada9d" |
|
333 |
}, |
|
334 |
{ |
|
335 |
"startKey": "50|dedup_wf_001::7f03900f6f068d95d144cca696eada9d", |
|
336 |
"endKey": "50|dedup_wf_001::93700a5d375511cb40b86026850c0c04" |
|
337 |
}, |
|
338 |
{ |
|
339 |
"startKey": "50|dedup_wf_001::93700a5d375511cb40b86026850c0c04", |
|
340 |
"endKey": "50|dedup_wf_001::a91b3388333d2ce8543f6d18e9230184" |
|
341 |
}, |
|
342 |
{ |
|
343 |
"startKey": "50|dedup_wf_001::a91b3388333d2ce8543f6d18e9230184", |
|
344 |
"endKey": "50|dedup_wf_001::b5692a63f5561061fdf347d442c125c9" |
|
345 |
}, |
|
346 |
{ |
|
347 |
"startKey": "50|dedup_wf_001::b5692a63f5561061fdf347d442c125c9", |
|
348 |
"endKey": "50|dedup_wf_001::c23ccd4da7d56c6042ad153d101e3b4a" |
|
349 |
}, |
|
350 |
{ |
|
351 |
"startKey": "50|dedup_wf_001::c23ccd4da7d56c6042ad153d101e3b4a", |
|
352 |
"endKey": "50|dedup_wf_001::d9bd4adea34900b74b1083e0aa973d14" |
|
353 |
}, |
|
354 |
{ |
|
355 |
"startKey": "50|dedup_wf_001::d9bd4adea34900b74b1083e0aa973d14", |
|
356 |
"endKey": "50|dedup_wf_001::f3df01ae80f7ee559e45c682740526f6" |
|
357 |
}, |
|
358 |
{ |
|
359 |
"startKey": "50|dedup_wf_001::f3df01ae80f7ee559e45c682740526f6", |
|
360 |
"endKey": "50|doajarticles::114db5b70d6bb92281c0a48a3cdeab90" |
|
361 |
}, |
|
362 |
{ |
|
363 |
"startKey": "50|doajarticles::114db5b70d6bb92281c0a48a3cdeab90", |
|
364 |
"endKey": "50|doajarticles::308e70820d5eb34b379a27f44cec58eb" |
|
365 |
}, |
|
366 |
{ |
|
367 |
"startKey": "50|doajarticles::308e70820d5eb34b379a27f44cec58eb", |
|
368 |
"endKey": "50|doajarticles::52da1a571f4a5c558d4041f86d57a740" |
|
369 |
}, |
|
370 |
{ |
|
371 |
"startKey": "50|doajarticles::52da1a571f4a5c558d4041f86d57a740", |
|
372 |
"endKey": "50|doajarticles::70bb02eee5d62a2b0155934753b3d55b" |
|
373 |
}, |
|
374 |
{ |
|
375 |
"startKey": "50|doajarticles::70bb02eee5d62a2b0155934753b3d55b", |
|
376 |
"endKey": "50|doajarticles::90e46c2630de99bf901b6f9d5ccff508" |
|
377 |
}, |
|
378 |
{ |
|
379 |
"startKey": "50|doajarticles::90e46c2630de99bf901b6f9d5ccff508", |
|
380 |
"endKey": "50|doajarticles::b01eb463260993ab479ae998f29bcf7c" |
|
381 |
}, |
|
382 |
{ |
|
383 |
"startKey": "50|doajarticles::b01eb463260993ab479ae998f29bcf7c", |
|
384 |
"endKey": "50|doajarticles::cba7f3680eab0417b1f0c93a0319ba64" |
|
385 |
}, |
|
386 |
{ |
|
387 |
"startKey": "50|doajarticles::cba7f3680eab0417b1f0c93a0319ba64", |
|
388 |
"endKey": "50|doajarticles::e76f01bc97e39d5e220452928b0f01c0" |
|
389 |
}, |
|
390 |
{ |
|
391 |
"startKey": "50|doajarticles::e76f01bc97e39d5e220452928b0f01c0", |
|
392 |
"endKey": "50|fp7_env_____::55381a5c6aa3147c4b105ab4f1f341a6" |
|
393 |
}, |
|
394 |
{ |
|
395 |
"startKey": "50|fp7_env_____::55381a5c6aa3147c4b105ab4f1f341a6", |
|
396 |
"endKey": "50|od______1064::0db8b9d3d5b726dd486715a201819fe0" |
|
397 |
}, |
|
398 |
{ |
|
399 |
"startKey": "50|od______1064::0db8b9d3d5b726dd486715a201819fe0", |
|
400 |
"endKey": "50|od______1064::4135d9b247a3d7c2af4bec1360ef77ff" |
|
401 |
}, |
|
402 |
{ |
|
403 |
"startKey": "50|od______1064::4135d9b247a3d7c2af4bec1360ef77ff", |
|
404 |
"endKey": "50|od______1064::71cf5d71f20907e836a84b14085851fc" |
|
405 |
}, |
|
406 |
{ |
|
407 |
"startKey": "50|od______1064::71cf5d71f20907e836a84b14085851fc", |
|
408 |
"endKey": "50|od______1064::9b9c24d0e70c897be2b702184b4758fd" |
|
409 |
}, |
|
410 |
{ |
|
411 |
"startKey": "50|od______1064::9b9c24d0e70c897be2b702184b4758fd", |
|
412 |
"endKey": "50|od______1064::d4424e2a2d13074f01581de61395357d" |
|
413 |
}, |
|
414 |
{ |
|
415 |
"startKey": "50|od______1064::d4424e2a2d13074f01581de61395357d", |
|
416 |
"endKey": "50|od______1064::ffd165b7795b6a6386fa7a5b2eb56e18" |
|
417 |
}, |
|
418 |
{ |
|
419 |
"startKey": "50|od______1064::ffd165b7795b6a6386fa7a5b2eb56e18", |
|
420 |
"endKey": "50|od______1201::7a8d284e224d73834f7ff0415abb53a6" |
|
421 |
}, |
|
422 |
{ |
|
423 |
"startKey": "50|od______1201::7a8d284e224d73834f7ff0415abb53a6", |
|
424 |
"endKey": "50|od______1272::db9ab1802259f3aa3f37f982b7f2b130" |
|
425 |
}, |
|
426 |
{ |
|
427 |
"startKey": "50|od______1272::db9ab1802259f3aa3f37f982b7f2b130", |
|
428 |
"endKey": "50|od______1389::60314c89ce4de7fa72cfb2fdad10e1b7" |
|
429 |
}, |
|
430 |
{ |
|
431 |
"startKey": "50|od______1389::60314c89ce4de7fa72cfb2fdad10e1b7", |
|
432 |
"endKey": "50|od______1502::f9401983425dee941b3021b59ef606fc" |
|
433 |
}, |
|
434 |
{ |
|
435 |
"startKey": "50|od______1502::f9401983425dee941b3021b59ef606fc", |
|
436 |
"endKey": "50|od______1687::e5fc94cd56e8bf70c59e67e7871b6250" |
|
437 |
}, |
|
438 |
{ |
|
439 |
"startKey": "50|od______1687::e5fc94cd56e8bf70c59e67e7871b6250", |
|
440 |
"endKey": "50|od______2001::9d1cdaaa9fc6f5fcc7bc7b5c4e547ff4" |
|
441 |
}, |
|
442 |
{ |
|
443 |
"startKey": "50|od______2001::9d1cdaaa9fc6f5fcc7bc7b5c4e547ff4", |
|
444 |
"endKey": "50|od______3061::4173eaee46b73cd5f43d2417d54e4ebe" |
|
445 |
}, |
|
446 |
{ |
|
447 |
"startKey": "50|od______3061::4173eaee46b73cd5f43d2417d54e4ebe", |
|
448 |
"endKey": "50|od_______162::5336859ed82fab129bfffcd2d8bb9806" |
|
449 |
}, |
|
450 |
{ |
|
451 |
"startKey": "50|od_______162::5336859ed82fab129bfffcd2d8bb9806", |
|
452 |
"endKey": "50|od_______165::14a78f7d1f6a358122eb3348e9a2f8b2" |
|
453 |
}, |
|
454 |
{ |
|
455 |
"startKey": "50|od_______165::14a78f7d1f6a358122eb3348e9a2f8b2", |
|
456 |
"endKey": "50|od_______165::4a81f0ac988218c4805275db0809ee87" |
|
457 |
}, |
|
458 |
{ |
|
459 |
"startKey": "50|od_______165::4a81f0ac988218c4805275db0809ee87", |
|
460 |
"endKey": "50|od_______165::987fa6a514917d6d503087b85e205f7d" |
|
461 |
}, |
|
462 |
{ |
|
463 |
"startKey": "50|od_______165::987fa6a514917d6d503087b85e205f7d", |
|
464 |
"endKey": "50|od_______165::d63335b5b0d6d8acb92840c371fb02fe" |
|
465 |
}, |
|
466 |
{ |
|
467 |
"startKey": "50|od_______165::d63335b5b0d6d8acb92840c371fb02fe", |
|
468 |
"endKey": "50|od_______206::f300a704c38f523b3fcec2a2b282ee6e" |
|
469 |
}, |
|
470 |
{ |
|
471 |
"startKey": "50|od_______206::f300a704c38f523b3fcec2a2b282ee6e", |
|
472 |
"endKey": "50|od_______322::7c58d6a5d5a9cda35bacf1272834cb26" |
|
473 |
}, |
|
474 |
{ |
|
475 |
"startKey": "50|od_______322::7c58d6a5d5a9cda35bacf1272834cb26", |
|
476 |
"endKey": "50|od_______369::d6e7494e5141d30a56e2eebf7f5ee33d" |
|
477 |
}, |
|
478 |
{ |
|
479 |
"startKey": "50|od_______369::d6e7494e5141d30a56e2eebf7f5ee33d", |
|
480 |
"endKey": "50|od_______540::28c6e29fd1fbe925c74ef15f3462f83d" |
|
481 |
}, |
|
482 |
{ |
|
483 |
"startKey": "50|od_______540::28c6e29fd1fbe925c74ef15f3462f83d", |
|
484 |
"endKey": "50|od_______642::a62315284dad20594bf02b6705b1ff49" |
|
485 |
}, |
|
486 |
{ |
|
487 |
"startKey": "50|od_______642::a62315284dad20594bf02b6705b1ff49", |
|
488 |
"endKey": "50|od_______908::01bde75cb624a9f54558cfc630fab270" |
|
489 |
}, |
|
490 |
{ |
|
491 |
"startKey": "50|od_______908::01bde75cb624a9f54558cfc630fab270", |
|
492 |
"endKey": "50|od_______908::0ce7e68c6e1e3ec87e3837569a7164c5" |
|
493 |
}, |
|
494 |
{ |
|
495 |
"startKey": "50|od_______908::0ce7e68c6e1e3ec87e3837569a7164c5", |
|
496 |
"endKey": "50|od_______908::178c6c5809f3a7ac7789ff97cf7bdea6" |
|
497 |
}, |
|
498 |
{ |
|
499 |
"startKey": "50|od_______908::178c6c5809f3a7ac7789ff97cf7bdea6", |
|
500 |
"endKey": "50|od_______908::225596f36d88fc945720ae38026127be" |
|
501 |
}, |
|
502 |
{ |
|
503 |
"startKey": "50|od_______908::225596f36d88fc945720ae38026127be", |
|
504 |
"endKey": "50|od_______908::2f2a95b64aac1cf130bc5262840663cc" |
|
505 |
}, |
|
506 |
{ |
|
507 |
"startKey": "50|od_______908::2f2a95b64aac1cf130bc5262840663cc", |
|
508 |
"endKey": "50|od_______908::3cd3b7d891c22944507fa04dabffe014" |
|
509 |
}, |
|
510 |
{ |
|
511 |
"startKey": "50|od_______908::3cd3b7d891c22944507fa04dabffe014", |
|
512 |
"endKey": "50|od_______908::49829a5e128c0179d6a97705c8e62325" |
|
513 |
}, |
|
514 |
{ |
|
515 |
"startKey": "50|od_______908::49829a5e128c0179d6a97705c8e62325", |
|
516 |
"endKey": "50|od_______908::58e11440d1bd36196ce307b73a1ceb3f" |
|
517 |
}, |
|
518 |
{ |
|
519 |
"startKey": "50|od_______908::58e11440d1bd36196ce307b73a1ceb3f", |
|
520 |
"endKey": "50|od_______908::67cf9887d98ee7bd160a44f6bed53ae3" |
|
521 |
}, |
|
522 |
{ |
|
523 |
"startKey": "50|od_______908::67cf9887d98ee7bd160a44f6bed53ae3", |
|
524 |
"endKey": "50|od_______908::78c37e55717fca99a50b346427b93e73" |
|
525 |
}, |
|
526 |
{ |
|
527 |
"startKey": "50|od_______908::78c37e55717fca99a50b346427b93e73", |
|
528 |
"endKey": "50|od_______908::88dc399017a984de9e2272c8b67c3a37" |
|
529 |
}, |
|
530 |
{ |
|
531 |
"startKey": "50|od_______908::88dc399017a984de9e2272c8b67c3a37", |
|
532 |
"endKey": "50|od_______908::99c9b13da30b6de059089e1573e3e5d9" |
|
533 |
}, |
|
534 |
{ |
|
535 |
"startKey": "50|od_______908::99c9b13da30b6de059089e1573e3e5d9", |
|
536 |
"endKey": "50|od_______908::ab14999d98f71e580f169eff61bd0064" |
|
537 |
}, |
|
538 |
{ |
|
539 |
"startKey": "50|od_______908::ab14999d98f71e580f169eff61bd0064", |
|
540 |
"endKey": "50|od_______908::bf8f7155a5c180418e9d31815b2a8461" |
|
541 |
}, |
|
542 |
{ |
|
543 |
"startKey": "50|od_______908::bf8f7155a5c180418e9d31815b2a8461", |
|
544 |
"endKey": "50|od_______908::d0c3b0de981570db3c41d11bc015400c" |
|
545 |
}, |
|
546 |
{ |
|
547 |
"startKey": "50|od_______908::d0c3b0de981570db3c41d11bc015400c", |
|
548 |
"endKey": "50|od_______908::e33858348d1f218d7233b5b0ce2ab2f6" |
|
549 |
}, |
|
550 |
{ |
|
551 |
"startKey": "50|od_______908::e33858348d1f218d7233b5b0ce2ab2f6", |
|
552 |
"endKey": "50|od_______908::f669e13899aef0eca6d7c45e6cb3fcad" |
|
553 |
}, |
|
554 |
{ |
|
555 |
"startKey": "50|od_______908::f669e13899aef0eca6d7c45e6cb3fcad", |
|
556 |
"endKey": "50|od_______970::e7887b56b0cdab4759f0daaf482f8e77" |
|
557 |
}, |
|
558 |
{ |
|
559 |
"startKey": "50|od_______970::e7887b56b0cdab4759f0daaf482f8e77", |
|
560 |
"endKey": "50|od________18::29fcec61e938883d1edee2bd4b1b247e" |
|
561 |
}, |
|
562 |
{ |
|
563 |
"startKey": "50|od________18::29fcec61e938883d1edee2bd4b1b247e", |
|
564 |
"endKey": "50|od________18::56cc9edd064372e6f3f81e9576cac953" |
|
565 |
}, |
|
566 |
{ |
|
567 |
"startKey": "50|od________18::56cc9edd064372e6f3f81e9576cac953", |
|
568 |
"endKey": "50|od________18::879d332de7942af09ff782d3fbb757b0" |
|
569 |
}, |
|
570 |
{ |
|
571 |
"startKey": "50|od________18::879d332de7942af09ff782d3fbb757b0", |
|
572 |
"endKey": "50|od________18::b46caa47ef0cfd7bc38335c4143ff766" |
|
573 |
}, |
|
574 |
{ |
|
575 |
"startKey": "50|od________18::b46caa47ef0cfd7bc38335c4143ff766", |
|
576 |
"endKey": "50|od________18::deb18d84795807a91e4276c0eb146d03" |
|
577 |
}, |
|
578 |
{ |
|
579 |
"startKey": "50|od________18::deb18d84795807a91e4276c0eb146d03", |
|
580 |
"endKey": "50|od________18::fe9cea245af7bc693e39f349c163e051" |
|
581 |
}, |
|
582 |
{ |
|
583 |
"startKey": "50|od________18::fe9cea245af7bc693e39f349c163e051", |
|
584 |
"endKey": "" |
|
585 |
} |
|
586 |
] |
|
587 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/HadoopBlackboardActions.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi; |
|
2 |
|
|
3 |
/** |
|
4 |
* Enum of the available HadoopBlackboardActions. |
|
5 |
*/ |
|
6 |
public enum HadoopBlackboardActions { |
|
7 |
|
|
8 |
/** The submit mapreduce job. */ |
|
9 |
SUBMIT_MAPREDUCE_JOB, |
|
10 |
/** The submit admin job. */ |
|
11 |
SUBMIT_ADMIN_JOB, |
|
12 |
/** The kill job. */ |
|
13 |
KILL_JOB, |
|
14 |
/** The submit oozie job. */ |
|
15 |
SUBMIT_OOZIE_JOB, |
|
16 |
/** The import epr to hbase job. */ |
|
17 |
IMPORT_EPR_HBASE, |
|
18 |
/** The delete epr from hbase job */ |
|
19 |
DELETE_EPR_HBASE, |
|
20 |
/** The import epr to hdfs job */ |
|
21 |
IMPORT_EPR_HDFS, |
|
22 |
/** The delete hdfs path. */ |
|
23 |
DELETE_HDFS_PATH, |
|
24 |
/** The create hdfs directory. */ |
|
25 |
CREATE_HDFS_DIR |
|
26 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/hbase/schema/HBaseTableDescriptor.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase.schema; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Set; |
|
5 |
|
|
6 |
import com.google.gson.Gson; |
|
7 |
|
|
8 |
/** |
|
9 |
* Created by claudio on 11/11/15. |
|
10 |
*/ |
|
11 |
public class HBaseTableDescriptor { |
|
12 |
|
|
13 |
private Set<String> columns; |
|
14 |
|
|
15 |
private List<HBaseTableRegionInfo> regions; |
|
16 |
|
|
17 |
public HBaseTableDescriptor() { |
|
18 |
} |
|
19 |
|
|
20 |
public HBaseTableDescriptor(final Set<String> columns, final List<HBaseTableRegionInfo> regions) { |
|
21 |
this.columns = columns; |
|
22 |
this.regions = regions; |
|
23 |
} |
|
24 |
|
|
25 |
public static HBaseTableDescriptor fromJSON(String json) { |
|
26 |
return new Gson().fromJson(json, HBaseTableDescriptor.class); |
|
27 |
} |
|
28 |
|
|
29 |
public Set<String> getColumns() { |
|
30 |
return columns; |
|
31 |
} |
|
32 |
|
|
33 |
public void setColumns(final Set<String> columns) { |
|
34 |
this.columns = columns; |
|
35 |
} |
|
36 |
|
|
37 |
public List<HBaseTableRegionInfo> getRegions() { |
|
38 |
return regions; |
|
39 |
} |
|
40 |
|
|
41 |
public void setRegions(final List<HBaseTableRegionInfo> regions) { |
|
42 |
this.regions = regions; |
|
43 |
} |
|
44 |
|
|
45 |
@Override |
|
46 |
public String toString() { |
|
47 |
return new Gson().toJson(this); |
|
48 |
} |
|
49 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/hbase/schema/HBaseTableRegionInfo.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase.schema; |
|
2 |
|
|
3 |
import com.google.gson.Gson; |
|
4 |
|
|
5 |
/** |
|
6 |
* Created by claudio on 11/11/15. |
|
7 |
*/ |
|
8 |
public class HBaseTableRegionInfo { |
|
9 |
|
|
10 |
private String startKey; |
|
11 |
private String endKey; |
|
12 |
|
|
13 |
public HBaseTableRegionInfo() { |
|
14 |
} |
|
15 |
|
|
16 |
public HBaseTableRegionInfo(final String startKey, final String endKey) { |
|
17 |
this.startKey = startKey; |
|
18 |
this.endKey = endKey; |
|
19 |
} |
|
20 |
|
|
21 |
public String getStartKey() { |
|
22 |
return startKey; |
|
23 |
} |
|
24 |
|
|
25 |
public void setStartKey(final String startKey) { |
|
26 |
this.startKey = startKey; |
|
27 |
} |
|
28 |
|
|
29 |
public String getEndKey() { |
|
30 |
return endKey; |
|
31 |
} |
|
32 |
|
|
33 |
public void setEndKey(final String endKey) { |
|
34 |
this.endKey = endKey; |
|
35 |
} |
|
36 |
|
|
37 |
@Override |
|
38 |
public String toString() { |
|
39 |
return new Gson().toJson(this); |
|
40 |
} |
|
41 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/hbase/HBaseRowDescriptor.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import javax.xml.bind.annotation.XmlRootElement; |
|
5 |
|
|
6 |
import com.google.common.collect.Lists; |
|
7 |
import com.google.gson.Gson; |
|
8 |
|
|
9 |
/** |
|
10 |
* Created by claudio on 23/10/15. |
|
11 |
*/ |
|
12 |
@XmlRootElement |
|
13 |
public class HBaseRowDescriptor { |
|
14 |
|
|
15 |
private String rowKey; |
|
16 |
|
|
17 |
private List<Column> columns; |
|
18 |
|
|
19 |
public HBaseRowDescriptor() { |
|
20 |
columns = Lists.newArrayList(); |
|
21 |
} |
|
22 |
|
|
23 |
public HBaseRowDescriptor(String rowKey, List<Column> columns) { |
|
24 |
this.rowKey = rowKey; |
|
25 |
this.columns = columns; |
|
26 |
} |
|
27 |
|
|
28 |
public static HBaseRowDescriptor fromJSON(String json) { |
|
29 |
return new Gson().fromJson(json, HBaseRowDescriptor.class); |
|
30 |
} |
|
31 |
|
|
32 |
public String getRowKey() { |
|
33 |
return rowKey; |
|
34 |
} |
|
35 |
|
|
36 |
public List<Column> getColumns() { |
|
37 |
return columns; |
|
38 |
} |
|
39 |
|
|
40 |
public void setRowKey(final String rowKey) { |
|
41 |
this.rowKey = rowKey; |
|
42 |
} |
|
43 |
|
|
44 |
public void setColumns(final List<Column> columns) { |
|
45 |
this.columns = columns; |
|
46 |
} |
|
47 |
|
|
48 |
@Override |
|
49 |
public String toString() { |
|
50 |
return new Gson().toJson(this); |
|
51 |
} |
|
52 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/hbase/Column.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi.hbase; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import javax.xml.bind.annotation.XmlRootElement; |
|
5 |
|
|
6 |
import com.google.common.collect.Lists; |
|
7 |
import com.google.gson.Gson; |
|
8 |
|
|
9 |
/** |
|
10 |
* Created by claudio on 23/10/15. |
|
11 |
*/ |
|
12 |
@XmlRootElement |
|
13 |
public class Column { |
|
14 |
|
|
15 |
private String family; |
|
16 |
|
|
17 |
private List<String> qualifier; |
|
18 |
|
|
19 |
public Column() { |
|
20 |
this.qualifier = Lists.newArrayList(); |
|
21 |
} |
|
22 |
|
|
23 |
public Column(final String family) { |
|
24 |
this.family = family; |
|
25 |
this.qualifier = Lists.newArrayList(); |
|
26 |
} |
|
27 |
|
|
28 |
public Column(final String family, final List<String> qualifier) { |
|
29 |
this.family = family; |
|
30 |
this.qualifier = qualifier; |
|
31 |
} |
|
32 |
|
|
33 |
public String getFamily() { |
|
34 |
return family; |
|
35 |
} |
|
36 |
|
|
37 |
public void setFamily(final String family) { |
|
38 |
this.family = family; |
|
39 |
} |
|
40 |
|
|
41 |
public List<String> getQualifier() { |
|
42 |
return qualifier; |
|
43 |
} |
|
44 |
|
|
45 |
public void setQualifier(final List<String> qualifier) { |
|
46 |
this.qualifier = qualifier; |
|
47 |
} |
|
48 |
|
|
49 |
@Override |
|
50 |
public String toString() { |
|
51 |
return new Gson().toJson(this); |
|
52 |
} |
|
53 |
|
|
54 |
|
|
55 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/HadoopJobDescriptor.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi; |
|
2 |
|
|
3 |
import javax.xml.bind.annotation.XmlRootElement; |
|
4 |
|
|
5 |
/** |
|
6 |
* <pre> |
|
7 |
* {@code |
|
8 |
* <hadoopJobDescriptor> |
|
9 |
* <jobName>ImportRecordsJob</jobName> |
|
10 |
* <jobDescription>map only job that maps xml metada records from a sequence file into an hbase table</jobDescription> |
|
11 |
* <id>abc</id> |
|
12 |
* <status>xyz</status> |
|
13 |
* </hadoopJobDescriptor> |
|
14 |
* } |
|
15 |
* </pre> |
|
16 |
* |
|
17 |
* @author claudio |
|
18 |
*/ |
|
19 |
@XmlRootElement |
|
20 |
public class HadoopJobDescriptor { |
|
21 |
|
|
22 |
/** The id. */ |
|
23 |
private String id; |
|
24 |
|
|
25 |
/** The job name. */ |
|
26 |
private String jobName; |
|
27 |
|
|
28 |
/** The job description. */ |
|
29 |
private String jobDescription; |
|
30 |
|
|
31 |
/** The status. */ |
|
32 |
private String status; |
|
33 |
|
|
34 |
/** The start time. */ |
|
35 |
private String startTime; |
|
36 |
|
|
37 |
/** The last activity. */ |
|
38 |
private String lastActivity; |
|
39 |
|
|
40 |
/** The hadoop id. */ |
|
41 |
private String hadoopId; |
|
42 |
|
|
43 |
/** The tracking url. */ |
|
44 |
private String trackingUrl; |
|
45 |
|
|
46 |
/** |
|
47 |
* Instantiates a new hadoop job descriptor. |
|
48 |
*/ |
|
49 |
public HadoopJobDescriptor() {} |
|
50 |
|
|
51 |
/** |
|
52 |
* Instantiates a new hadoop job descriptor. |
|
53 |
* |
|
54 |
* @param jobName |
|
55 |
* the job name |
|
56 |
* @param jobDescription |
|
57 |
* the job description |
|
58 |
* @param id |
|
59 |
* the id |
|
60 |
* @param status |
|
61 |
* the status |
|
62 |
* @param startTime |
|
63 |
* the start time |
|
64 |
* @param lastActivity |
|
65 |
* the last activity |
|
66 |
* @param hadoopId |
|
67 |
* the hadoop id |
|
68 |
* @param trackingUrl |
|
69 |
* the tracking url |
|
70 |
*/ |
|
71 |
public HadoopJobDescriptor(final String jobName, final String jobDescription, final String id, final String status, final String startTime, |
|
72 |
final String lastActivity, final String hadoopId, final String trackingUrl) { |
|
73 |
super(); |
|
74 |
this.id = id; |
|
75 |
this.jobName = jobName; |
|
76 |
this.jobDescription = jobDescription; |
|
77 |
this.status = status; |
|
78 |
this.startTime = startTime; |
|
79 |
this.lastActivity = lastActivity; |
|
80 |
this.hadoopId = hadoopId; |
|
81 |
this.trackingUrl = trackingUrl; |
|
82 |
} |
|
83 |
|
|
84 |
/** |
|
85 |
* Gets the job name. |
|
86 |
* |
|
87 |
* @return the job name |
|
88 |
*/ |
|
89 |
public String getJobName() { |
|
90 |
return jobName; |
|
91 |
} |
|
92 |
|
|
93 |
/** |
|
94 |
* Sets the job name. |
|
95 |
* |
|
96 |
* @param jobName |
|
97 |
* the new job name |
|
98 |
*/ |
|
99 |
public void setJobName(final String jobName) { |
|
100 |
this.jobName = jobName; |
|
101 |
} |
|
102 |
|
|
103 |
/** |
|
104 |
* Gets the id. |
|
105 |
* |
|
106 |
* @return the id |
|
107 |
*/ |
|
108 |
public String getId() { |
|
109 |
return id; |
|
110 |
} |
|
111 |
|
|
112 |
/** |
|
113 |
* Sets the id. |
|
114 |
* |
|
115 |
* @param id |
|
116 |
* the new id |
|
117 |
*/ |
|
118 |
public void setId(final String id) { |
|
119 |
this.id = id; |
|
120 |
} |
|
121 |
|
|
122 |
/** |
|
123 |
* Gets the job description. |
|
124 |
* |
|
125 |
* @return the job description |
|
126 |
*/ |
|
127 |
public String getJobDescription() { |
|
128 |
return jobDescription; |
|
129 |
} |
|
130 |
|
|
131 |
/** |
|
132 |
* Sets the job description. |
|
133 |
* |
|
134 |
* @param jobDescription |
|
135 |
* the new job description |
|
136 |
*/ |
|
137 |
public void setJobDescription(final String jobDescription) { |
|
138 |
this.jobDescription = jobDescription; |
|
139 |
} |
|
140 |
|
|
141 |
/** |
|
142 |
* Gets the status. |
|
143 |
* |
|
144 |
* @return the status |
|
145 |
*/ |
|
146 |
public String getStatus() { |
|
147 |
return status; |
|
148 |
} |
|
149 |
|
|
150 |
/** |
|
151 |
* Sets the status. |
|
152 |
* |
|
153 |
* @param status |
|
154 |
* the new status |
|
155 |
*/ |
|
156 |
public void setStatus(final String status) { |
|
157 |
this.status = status; |
|
158 |
} |
|
159 |
|
|
160 |
/** |
|
161 |
* Gets the start time. |
|
162 |
* |
|
163 |
* @return the start time |
|
164 |
*/ |
|
165 |
public String getStartTime() { |
|
166 |
return startTime; |
|
167 |
} |
|
168 |
|
|
169 |
/** |
|
170 |
* Sets the start time. |
|
171 |
* |
|
172 |
* @param startTime |
|
173 |
* the new start time |
|
174 |
*/ |
|
175 |
public void setStartTime(final String startTime) { |
|
176 |
this.startTime = startTime; |
|
177 |
} |
|
178 |
|
|
179 |
/** |
|
180 |
* Gets the last activity. |
|
181 |
* |
|
182 |
* @return the last activity |
|
183 |
*/ |
|
184 |
public String getLastActivity() { |
|
185 |
return lastActivity; |
|
186 |
} |
|
187 |
|
|
188 |
/** |
|
189 |
* Sets the last activity. |
|
190 |
* |
|
191 |
* @param lastActivity |
|
192 |
* the new last activity |
|
193 |
*/ |
|
194 |
public void setLastActivity(final String lastActivity) { |
|
195 |
this.lastActivity = lastActivity; |
|
196 |
} |
|
197 |
|
|
198 |
/** |
|
199 |
* Gets the hadoop id. |
|
200 |
* |
|
201 |
* @return the hadoop id |
|
202 |
*/ |
|
203 |
public String getHadoopId() { |
|
204 |
return hadoopId; |
|
205 |
} |
|
206 |
|
|
207 |
/** |
|
208 |
* Sets the hadoop id. |
|
209 |
* |
|
210 |
* @param hadoopId |
|
211 |
* the new hadoop id |
|
212 |
*/ |
|
213 |
public void setHadoopId(final String hadoopId) { |
|
214 |
this.hadoopId = hadoopId; |
|
215 |
} |
|
216 |
|
|
217 |
/** |
|
218 |
* Gets the tracking url. |
|
219 |
* |
|
220 |
* @return the tracking url |
|
221 |
*/ |
|
222 |
public String getTrackingUrl() { |
|
223 |
return trackingUrl; |
|
224 |
} |
|
225 |
|
|
226 |
/** |
|
227 |
* Sets the tracking url. |
|
228 |
* |
|
229 |
* @param trackingUrl |
|
230 |
* the new tracking url |
|
231 |
*/ |
|
232 |
public void setTrackingUrl(final String trackingUrl) { |
|
233 |
this.trackingUrl = trackingUrl; |
|
234 |
} |
|
235 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/HadoopJobType.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi; |
|
2 |
|
|
3 |
import java.util.Arrays; |
|
4 |
import java.util.List; |
|
5 |
import java.util.Set; |
|
6 |
|
|
7 |
import com.google.common.base.Function; |
|
8 |
import com.google.common.collect.Iterables; |
|
9 |
import com.google.common.collect.Lists; |
|
10 |
import com.google.common.collect.Sets; |
|
11 |
|
|
12 |
/** |
|
13 |
* Enum of supported hadoop job types. |
|
14 |
* |
|
15 |
* @author claudio |
|
16 |
*/ |
|
17 |
public enum HadoopJobType { |
|
18 |
|
|
19 |
/** The mapreduce. */ |
|
20 |
mapreduce, |
|
21 |
|
|
22 |
/** The admin. */ |
|
23 |
admin, |
|
24 |
|
|
25 |
/** The oozie. */ |
|
26 |
oozie; |
|
27 |
|
|
28 |
/** |
|
29 |
* The Enum AdminJobType. |
|
30 |
*/ |
|
31 |
public enum AdminJobType { |
|
32 |
|
|
33 |
/** |
|
34 |
* The copytable job. |
|
35 |
* |
|
36 |
* <pre> |
|
37 |
* {@code |
|
38 |
* bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable --starttime=1265875194289 --endtime=1265878794289 |
|
39 |
* --peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3 TestTable <code> |
|
40 |
* |
|
41 |
* Options: |
|
42 |
* rs.class hbase.regionserver.class of the peer cluster, |
|
43 |
* specify if different from current cluster |
|
44 |
* rs.impl hbase.regionserver.impl of the peer cluster, |
|
45 |
* startrow the start row |
|
46 |
* stoprow the stop row |
|
47 |
* starttime beginning of the time range (unixtime in millis) |
|
48 |
* without endtime means from starttime to forever |
|
49 |
* endtime end of the time range. Ignored if no starttime specified. |
|
50 |
* versions number of cell versions to copy |
|
51 |
* new.name new table's name |
|
52 |
* peer.adr Address of the peer cluster given in the format |
|
53 |
* hbase.zookeeer.quorum:hbase.zookeeper.client.port:zookeeper.znode.parent |
|
54 |
* families comma-separated list of families to copy |
|
55 |
* To copy from cf1 to cf2, give sourceCfName:destCfName. |
|
56 |
* To keep the same name, just give "cfName" |
|
57 |
* all.cells also copy delete markers and deleted cells |
|
58 |
* |
|
59 |
* Args: |
|
60 |
* tablename Name of the table to copy |
|
61 |
* } |
|
62 |
* </pre> |
|
63 |
* |
|
64 |
* */ |
|
65 |
copytable; |
|
66 |
|
|
67 |
public static Set<String> asStringSet() { |
|
68 |
return Sets.newHashSet(asStringList()); |
|
69 |
} |
|
70 |
|
|
71 |
public static List<String> asStringList() { |
|
72 |
return Lists.newArrayList(Iterables.transform(asList(), new Function<AdminJobType, String>() { |
|
73 |
|
|
74 |
@Override |
|
75 |
public String apply(final AdminJobType type) { |
|
76 |
return type.toString(); |
|
77 |
} |
|
78 |
})); |
|
79 |
} |
|
80 |
|
|
81 |
public static List<AdminJobType> asList() { |
|
82 |
return Arrays.asList(AdminJobType.values()); |
|
83 |
} |
|
84 |
|
|
85 |
} |
|
86 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/HadoopServiceException.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi; |
|
2 |
|
|
3 |
import eu.dnetlib.common.rmi.RMIException; |
|
4 |
|
|
5 |
/** |
|
6 |
* The Class HadoopServiceException. |
|
7 |
*/ |
|
8 |
public class HadoopServiceException extends RMIException { |
|
9 |
|
|
10 |
/** The Constant serialVersionUID. */ |
|
11 |
private static final long serialVersionUID = 7523999812098059764L; |
|
12 |
|
|
13 |
/** |
|
14 |
* Instantiates a new hadoop service exception. |
|
15 |
* |
|
16 |
* @param string |
|
17 |
* the string |
|
18 |
*/ |
|
19 |
public HadoopServiceException(final String string) { |
|
20 |
super(string); |
|
21 |
} |
|
22 |
|
|
23 |
/** |
|
24 |
* Instantiates a new hadoop service exception. |
|
25 |
* |
|
26 |
* @param string |
|
27 |
* the string |
|
28 |
* @param exception |
|
29 |
* the exception |
|
30 |
*/ |
|
31 |
public HadoopServiceException(final String string, final Throwable exception) { |
|
32 |
super(string, exception); |
|
33 |
} |
|
34 |
|
|
35 |
/** |
|
36 |
* Instantiates a new hadoop service exception. |
|
37 |
* |
|
38 |
* @param exception |
|
39 |
* the exception |
|
40 |
*/ |
|
41 |
public HadoopServiceException(final Throwable exception) { |
|
42 |
super(exception); |
|
43 |
} |
|
44 |
|
|
45 |
} |
modules/dnet-hadoop-service-rmi/trunk/src/main/java/eu/dnetlib/data/hadoop/rmi/HadoopService.java | ||
---|---|---|
1 |
package eu.dnetlib.data.hadoop.rmi; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Map; |
|
5 |
import java.util.Set; |
|
6 |
import javax.jws.WebParam; |
|
7 |
import javax.jws.WebService; |
|
8 |
|
|
9 |
import eu.dnetlib.common.rmi.BaseService; |
|
10 |
import eu.dnetlib.data.hadoop.rmi.hbase.HBaseRowDescriptor; |
|
11 |
|
|
12 |
/** |
|
13 |
* The Interface HadoopService. |
|
14 |
*/ |
|
15 |
@WebService(targetNamespace = "http://services.dnetlib.eu/") |
|
16 |
public interface HadoopService extends BaseService { |
|
17 |
|
|
18 |
/** |
|
19 |
* List available jobs. |
|
20 |
* |
|
21 |
* @return the list |
|
22 |
* @throws HadoopServiceException the hadoop service exception |
|
23 |
*/ |
|
24 |
List<String> listAvailableJobs() throws HadoopServiceException; |
|
25 |
|
|
26 |
/** |
|
27 |
* List jobs. |
|
28 |
* |
|
29 |
* @param clusterName the cluster name |
|
30 |
* @return the list |
|
31 |
* @throws HadoopServiceException the hadoop service exception |
|
32 |
*/ |
|
33 |
List<HadoopJobDescriptor> listJobs(@WebParam(name = "clusterName") String clusterName) throws HadoopServiceException; |
|
34 |
|
|
35 |
/** |
|
36 |
* Kill job. |
|
37 |
* |
|
38 |
* @param jobId the job id |
|
39 |
* @return true, if successful |
|
40 |
* @throws HadoopServiceException the hadoop service exception |
|
41 |
*/ |
|
42 |
boolean killJob(@WebParam(name = "id") String jobId) throws HadoopServiceException; |
|
43 |
|
|
44 |
/** |
|
45 |
* List hbase tables. |
|
46 |
* |
|
47 |
* @param clusterName the cluster name |
|
48 |
* @return the list |
|
49 |
* @throws HadoopServiceException the hadoop service exception |
|
50 |
*/ |
|
51 |
List<String> listHbaseTables(@WebParam(name = "clusterName") String clusterName) throws HadoopServiceException; |
|
52 |
|
|
53 |
/** |
|
54 |
* Ensure hbase table. |
|
55 |
* |
|
56 |
* @param clusterName the cluster name |
|
57 |
* @param tableName the table name |
|
58 |
* @param columns the columns |
|
59 |
* @return true, if successful |
|
60 |
* @throws HadoopServiceException the hadoop service exception |
|
61 |
*/ |
|
62 |
boolean ensureHbaseTable(@WebParam(name = "clusterName") String clusterName, |
|
63 |
@WebParam(name = "tableName") String tableName, |
|
64 |
@WebParam(name = "columns") Set<String> columns) throws HadoopServiceException; |
|
65 |
|
|
66 |
/** |
|
67 |
* Creates the hbase table. |
|
68 |
* |
|
69 |
* @param clusterName the cluster name |
|
70 |
* @param tableName the table name |
|
71 |
* @param columns the columns |
|
72 |
* @return true, if successful |
|
73 |
* @throws HadoopServiceException the hadoop service exception |
|
74 |
*/ |
|
75 |
boolean createHbaseTable(@WebParam(name = "clusterName") String clusterName, |
|
76 |
@WebParam(name = "tableName") String tableName, |
|
77 |
@WebParam(name = "columns") Set<String> columns) throws HadoopServiceException; |
|
78 |
|
|
79 |
/** |
|
80 |
* Creates the hbase table. |
|
81 |
* |
|
82 |
* @param clusterName the cluster name |
|
83 |
* @param tableName the table name |
|
84 |
* @param tableConfiguration the table configuration: columns and region splits |
|
85 |
* @return true, if successful |
|
86 |
* @throws HadoopServiceException the hadoop service exception |
|
87 |
*/ |
|
88 |
boolean createConfiguredHbaseTable(@WebParam(name = "clusterName") String clusterName, |
|
89 |
@WebParam(name = "tableName") String tableName, |
|
90 |
@WebParam(name = "tableConfiguration") String tableConfiguration) throws HadoopServiceException; |
|
91 |
|
|
92 |
/** |
|
93 |
* Truncate hbase table. |
|
94 |
* |
|
95 |
* @param clusterName the cluster name |
|
96 |
* @param tableName the table name |
|
97 |
* @return true, if successful |
|
98 |
* @throws HadoopServiceException the hadoop service exception |
|
99 |
*/ |
|
100 |
boolean truncateHbaseTable(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "tableName") String tableName) |
|
101 |
throws HadoopServiceException; |
|
102 |
|
|
103 |
/** |
|
104 |
* Drop hbase table. |
|
105 |
* |
|
106 |
* @param clusterName the cluster name |
|
107 |
* @param tableName the table name |
|
108 |
* @return true, if successful |
|
109 |
* @throws HadoopServiceException the hadoop service exception |
|
110 |
*/ |
|
111 |
boolean dropHbaseTable(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "tableName") String tableName) throws HadoopServiceException; |
|
112 |
|
|
113 |
/** |
|
114 |
* Exist hbase table. |
|
115 |
* |
|
116 |
* @param clusterName the cluster name |
|
117 |
* @param tableName the table name |
|
118 |
* @return true, if successful |
|
119 |
* @throws HadoopServiceException the hadoop service exception |
|
120 |
*/ |
|
121 |
boolean existHbaseTable(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "tableName") String tableName) throws HadoopServiceException; |
|
122 |
|
|
123 |
/** |
|
124 |
* Describe hbase table. |
|
125 |
* |
|
126 |
* @param clusterName the cluster name |
|
127 |
* @param tableName the table name |
|
128 |
* @return the list of columns defined for that table. |
|
129 |
* @throws HadoopServiceException the hadoop service exception |
|
130 |
*/ |
|
131 |
List<String> describeHbaseTable(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "tableName") String tableName) |
|
132 |
throws HadoopServiceException; |
|
133 |
|
|
134 |
/** |
|
135 |
* Gets the cluster configuration. |
|
136 |
* |
|
137 |
* @param clusterName the cluster name |
|
138 |
* @return the cluster configuration |
|
139 |
* @throws HadoopServiceException the hadoop service exception |
|
140 |
*/ |
|
141 |
Map<String, String> getClusterConfiguration(@WebParam(name = "clusterName") String clusterName) throws HadoopServiceException; |
|
142 |
|
|
143 |
/** |
|
144 |
* Deletes a path on HDFS |
|
145 |
* |
|
146 |
* @param clusterName the cluster name |
|
147 |
* @param path the path to be deleted |
|
148 |
* @return true in case of success |
|
149 |
* @throws HadoopServiceException |
|
150 |
*/ |
|
151 |
boolean deleteHdfsPath(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "path") String path) |
|
152 |
throws HadoopServiceException; |
|
153 |
|
|
154 |
/** |
|
155 |
* Check if path on HDFS exists |
|
156 |
* |
|
157 |
* @param clusterName the cluster name |
|
158 |
* @param path the path to be deleted |
|
159 |
* @return true in case the path exists |
|
160 |
* @throws HadoopServiceException |
|
161 |
*/ |
|
162 |
boolean existHdfsPath(@WebParam(name = "clusterName") String clusterName, @WebParam(name = "path") String path) |
|
163 |
throws HadoopServiceException; |
|
164 |
|
|
165 |
/** |
|
166 |
* Creates a directory on HDFS, eventually deleteing its content. |
|
167 |
* |
|
168 |
* @param clusterName the cluster name |
|
169 |
* @param path the path to be created |
|
170 |
* @param force force delete before create |
|
171 |
* @return true in case of success |
|
172 |
* @throws HadoopServiceException |
|
173 |
*/ |
|
174 |
boolean createHdfsDirectory(@WebParam(name = "clusterName") String clusterName, |
|
175 |
@WebParam(name = "path") String path, |
|
176 |
@WebParam(name = "force") boolean force) |
|
177 |
throws HadoopServiceException; |
|
178 |
|
|
179 |
/** |
|
180 |
* List the configured cluster names. |
|
181 |
* |
|
182 |
* @return the list of configured cluster names. |
|
183 |
* @throws HadoopServiceException shouldn't happen |
|
184 |
*/ |
|
185 |
List<String> listClusters() throws HadoopServiceException; |
Also available in: Unified diff
codebase used to migrate to java8 the production system