Revision 45182
Added by Claudio Atzori over 7 years ago
modules/dnet-gridfs-objectstore/trunk/deploy.info | ||
---|---|---|
1 |
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-gridfs-objectstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-gridfs-objectstore"} |
modules/dnet-gridfs-objectstore/trunk/src/test/java/eu/dnetlib/test/TestQuery.java | ||
---|---|---|
1 |
package eu.dnetlib.test; |
|
2 |
|
|
3 |
import com.google.common.collect.Lists; |
|
4 |
import com.mongodb.*; |
|
5 |
import com.mongodb.gridfs.GridFS; |
|
6 |
import eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl; |
|
7 |
import org.junit.Ignore; |
|
8 |
import org.junit.Test; |
|
9 |
|
|
10 |
import java.io.UnsupportedEncodingException; |
|
11 |
import java.net.URLEncoder; |
|
12 |
import java.net.UnknownHostException; |
|
13 |
import java.util.ArrayList; |
|
14 |
import java.util.Date; |
|
15 |
|
|
16 |
public class TestQuery { |
|
17 |
|
|
18 |
@Test |
|
19 |
@Ignore |
|
20 |
public void testFromUntil() throws UnknownHostException, MongoException, UnsupportedEncodingException { |
|
21 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
22 |
Mongo mongo = new Mongo("localhost", 27017); |
|
23 |
DB db = mongo.getDB("objectStore"); |
|
24 |
dao.setDb(db); |
|
25 |
GridFS obs = new GridFS(db, "aeiouy"); |
|
26 |
BasicDBObject q = new BasicDBObject(); |
|
27 |
Date d = new Date(); |
|
28 |
|
|
29 |
Double from = new Double(d.getTime()); |
|
30 |
System.out.println(URLEncoder.encode("MY CRZY QUERY! +&+ :)", "UTF-8")); |
|
31 |
|
|
32 |
q.put("$lt", from.doubleValue()); |
|
33 |
|
|
34 |
DBCursor files = obs.getFileList(new BasicDBObject("metadata.timestamp", q)).sort(new BasicDBObject("metadata.id", 1)).skip(4).limit(3); |
|
35 |
ArrayList<DBObject> p = Lists.newArrayList((Iterable<DBObject>) files); |
|
36 |
for (DBObject tmp : p) { |
|
37 |
DBObject o = (DBObject) tmp.get("metadata"); |
|
38 |
System.out.println(o); |
|
39 |
} |
|
40 |
|
|
41 |
} |
|
42 |
} |
modules/dnet-gridfs-objectstore/trunk/src/test/java/eu/dnetlib/test/TestGridfs.java | ||
---|---|---|
1 |
package eu.dnetlib.test; |
|
2 |
|
|
3 |
import com.google.common.collect.Iterables; |
|
4 |
import com.google.common.collect.Lists; |
|
5 |
import com.mongodb.DB; |
|
6 |
import com.mongodb.Mongo; |
|
7 |
import com.mongodb.MongoException; |
|
8 |
import com.mongodb.gridfs.GridFS; |
|
9 |
import eu.dnetlib.data.objectstore.modular.ObjectBroker; |
|
10 |
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord; |
|
11 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore; |
|
12 |
import eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl; |
|
13 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile; |
|
14 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; |
|
15 |
import eu.dnetlib.data.objectstore.rmi.Protocols; |
|
16 |
import eu.dnetlib.enabling.resultset.ResultSetListener; |
|
17 |
import org.junit.After; |
|
18 |
import org.junit.Before; |
|
19 |
import org.junit.Ignore; |
|
20 |
import org.junit.Test; |
|
21 |
|
|
22 |
import java.io.FileInputStream; |
|
23 |
import java.io.FileNotFoundException; |
|
24 |
import java.net.UnknownHostException; |
|
25 |
import java.util.ArrayList; |
|
26 |
import java.util.List; |
|
27 |
import java.util.Scanner; |
|
28 |
|
|
29 |
public class TestGridfs { |
|
30 |
|
|
31 |
private static boolean deleted = false; |
|
32 |
|
|
33 |
@Before |
|
34 |
public void initialize() throws UnknownHostException, MongoException { |
|
35 |
if (deleted) return; |
|
36 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
37 |
Mongo mongo = new Mongo("localhost", 27017); |
|
38 |
DB db = mongo.getDB("objectStore"); |
|
39 |
dao.setDb(db); |
|
40 |
// dao.deleteObjectStore("aeiouy"); |
|
41 |
deleted = true; |
|
42 |
|
|
43 |
} |
|
44 |
|
|
45 |
@After |
|
46 |
public void showSize() throws UnknownHostException, MongoException, ObjectStoreServiceException { |
|
47 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
48 |
Mongo mongo = new Mongo("localhost", 27017); |
|
49 |
DB db = mongo.getDB("objectStore"); |
|
50 |
dao.setDb(db); |
|
51 |
ObjectStore obs = dao.getObjectStore("aeiouy"); |
|
52 |
System.out.println("Store size:" + obs.getSize()); |
|
53 |
|
|
54 |
} |
|
55 |
|
|
56 |
@Ignore |
|
57 |
@Test |
|
58 |
public void testFTP() throws FileNotFoundException, UnknownHostException, MongoException, ObjectStoreServiceException { |
|
59 |
System.out.println("Test FTP"); |
|
60 |
|
|
61 |
Scanner scanner = new Scanner(new FileInputStream("/home/sandro/orig_node2.txt"), "ascii"); |
|
62 |
List<String> results = new ArrayList<String>(); |
|
63 |
try { |
|
64 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
65 |
Mongo mongo = new Mongo("localhost", 27017); |
|
66 |
DB db = mongo.getDB("objectStore"); |
|
67 |
dao.setDb(db); |
|
68 |
for (int i = 0; i < 10; i++) { |
|
69 |
String currentURI = "ftp://node2.d.efg.research-infrastructures.eu/orig/" + scanner.nextLine(); |
|
70 |
ObjectStoreFile metadata = new ObjectStoreFile(); |
|
71 |
metadata.setAccessProtocol(Protocols.FTP); |
|
72 |
metadata.setURI(currentURI); |
|
73 |
metadata.setMimeType("jpeg"); |
|
74 |
metadata.setObjectID("" + i); |
|
75 |
results.add(metadata.toJSON()); |
|
76 |
|
|
77 |
} |
|
78 |
ObjectBroker o = new ObjectBroker(Protocols.HTTP, null, null, "p"); |
|
79 |
Iterable<ObjectStoreRecord> a = Iterables.transform(results, o); |
|
80 |
ObjectStore gfo = dao.getObjectStore("aeiouy"); |
|
81 |
|
|
82 |
gfo.feed(a, true); |
|
83 |
} finally { |
|
84 |
scanner.close(); |
|
85 |
} |
|
86 |
|
|
87 |
} |
|
88 |
|
|
89 |
@Test |
|
90 |
@Ignore |
|
91 |
public void testHTTP() throws FileNotFoundException, UnknownHostException, MongoException, ObjectStoreServiceException { |
|
92 |
|
|
93 |
Scanner scanner = new Scanner(new FileInputStream("/home/sandro/orig_node2.txt"), "ascii"); |
|
94 |
List<String> results = new ArrayList<String>(); |
|
95 |
System.out.println("Test HTTP"); |
|
96 |
try { |
|
97 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
98 |
Mongo mongo = new Mongo("localhost", 27017); |
|
99 |
DB db = mongo.getDB("objectStore"); |
|
100 |
dao.setDb(db); |
|
101 |
|
|
102 |
for (int i = 0; i < 50; i++) { |
|
103 |
|
|
104 |
String currentURI = "http://admin.d.efg.research-infrastructures.eu/orig/" + scanner.nextLine(); |
|
105 |
if (i > 10) { |
|
106 |
|
|
107 |
ObjectStoreFile metadata = new ObjectStoreFile(); |
|
108 |
metadata.setURI(currentURI); |
|
109 |
metadata.setObjectID("" + i); |
|
110 |
results.add(metadata.toJSON()); |
|
111 |
} |
|
112 |
|
|
113 |
} |
|
114 |
ObjectBroker o = new ObjectBroker(Protocols.HTTP, null, null, "jpeg"); |
|
115 |
Iterable<ObjectStoreRecord> a = Iterables.transform(results, o); |
|
116 |
ObjectStore gfo = dao.getObjectStore("aeiouy"); |
|
117 |
gfo.feed(a, true); |
|
118 |
} finally { |
|
119 |
scanner.close(); |
|
120 |
} |
|
121 |
|
|
122 |
} |
|
123 |
|
|
124 |
@Test |
|
125 |
@Ignore |
|
126 |
public void testFileSystem() throws FileNotFoundException, UnknownHostException, MongoException, ObjectStoreServiceException { |
|
127 |
|
|
128 |
System.out.println("Test File System"); |
|
129 |
List<String> results = new ArrayList<String>(); |
|
130 |
|
|
131 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
132 |
Mongo mongo = new Mongo("localhost", 27017); |
|
133 |
DB db = mongo.getDB("objectStore"); |
|
134 |
dao.setDb(db); |
|
135 |
|
|
136 |
for (int i = 50; i < 100; i++) { |
|
137 |
|
|
138 |
String currentURI = "/home/sandro/Desktop/download/" + i + ".jpg"; |
|
139 |
|
|
140 |
ObjectStoreFile metadata = new ObjectStoreFile(); |
|
141 |
metadata.setURI(currentURI); |
|
142 |
metadata.setObjectID("" + i); |
|
143 |
results.add(metadata.toJSON()); |
|
144 |
|
|
145 |
} |
|
146 |
ObjectBroker o = new ObjectBroker(Protocols.File_System, null, null, "jpeg"); |
|
147 |
Iterable<ObjectStoreRecord> a = Iterables.transform(results, o); |
|
148 |
ObjectStore gfo = dao.getObjectStore("aeiouy"); |
|
149 |
gfo.feed(a, true); |
|
150 |
|
|
151 |
} |
|
152 |
|
|
153 |
@Test |
|
154 |
@Ignore |
|
155 |
public void testDeliverObject() throws UnknownHostException, MongoException, ObjectStoreServiceException { |
|
156 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
157 |
Mongo mongo = new Mongo("localhost", 27017); |
|
158 |
DB db = mongo.getDB("objectStore"); |
|
159 |
dao.setDb(db); |
|
160 |
dao.setObjectStoreRESTURI("http://suca.org?id="); |
|
161 |
ObjectStore obs = dao.getObjectStore("aeiouy"); |
|
162 |
System.out.println("Deliver :" + obs.deliverObject("70")); |
|
163 |
} |
|
164 |
|
|
165 |
@Test |
|
166 |
@Ignore |
|
167 |
public void testDeliver() throws UnknownHostException, MongoException, ObjectStoreServiceException { |
|
168 |
GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
169 |
Mongo mongo = new Mongo("localhost", 27017); |
|
170 |
DB db = mongo.getDB("objectStore"); |
|
171 |
dao.setDb(db); |
|
172 |
dao.setObjectStoreRESTURI("http://suca.org/ab/cd?"); |
|
173 |
GridFS obs = new GridFS(db, "aeiouy"); |
|
174 |
List<String> records = Lists.newArrayList(); |
|
175 |
for (int i = 0; i < 20; i++) { |
|
176 |
records.add("" + i); |
|
177 |
} |
|
178 |
ObjectStore objectStore = dao.getObjectStore("aeiouy"); |
|
179 |
ResultSetListener rs = objectStore.deliverIds(records); |
|
180 |
System.out.println(rs.getSize()); |
|
181 |
List<String> res = rs.getResult(0, rs.getSize()); |
|
182 |
for (String r : res) { |
|
183 |
System.out.println(r); |
|
184 |
} |
|
185 |
|
|
186 |
// QueryBuilder qBuilder = |
|
187 |
// QueryBuilder.start("metadata.id").in(records); |
|
188 |
// |
|
189 |
// DBObject q = qBuilder.get(); |
|
190 |
// List<GridFSDBFile> out = obs.find(q); |
|
191 |
// UnaryFunction<String, GridFSDBFile> obtainURL = new |
|
192 |
// UnaryFunction<String, GridFSDBFile>() { |
|
193 |
// @Override |
|
194 |
// public String evaluate(GridFSDBFile input) { |
|
195 |
// |
|
196 |
// String baseURI = "http://suca.org?id="; |
|
197 |
// return baseURI + input.getMetaData().get("id"); |
|
198 |
// } |
|
199 |
// }; |
|
200 |
// List<String> mapped = MappedCollection.listMap(out, obtainURL); |
|
201 |
// |
|
202 |
// for (String ff : mapped) { |
|
203 |
// |
|
204 |
// System.out.println(ff); |
|
205 |
// } |
|
206 |
} |
|
207 |
// @Test |
|
208 |
// public void testFTP2() throws IOException, URISyntaxException { |
|
209 |
// URI uri = new URI( |
|
210 |
// "ftp://node2.d.efg.research-infrastructures.eu/orig/http___151.1.148.223_foto_high_AL_AL001_AL00000001.JPG"); |
|
211 |
// |
|
212 |
// Scanner scanner = new Scanner(new FileInputStream( |
|
213 |
// "/home/sandro/orig_node2.txt"), "ascii"); |
|
214 |
// List<String> results = new ArrayList<String>(); |
|
215 |
// try { |
|
216 |
// GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl(); |
|
217 |
// Mongo mongo = new Mongo("localhost", 27017); |
|
218 |
// DB db = mongo.getDB("objectStore"); |
|
219 |
// dao.setDb(db); |
|
220 |
// dao.deleteObjectStore("aeiouy"); |
|
221 |
// for (int i = 0; i < 100; i++) { |
|
222 |
// |
|
223 |
// String currentURI = |
|
224 |
// "http://admin.d.efg.research-infrastructures.eu/orig/" |
|
225 |
// + scanner.nextLine(); |
|
226 |
// if (i > 49) { |
|
227 |
// System.out.println("Download " + currentURI); |
|
228 |
// ObjectStoreFile metadata = new ObjectStoreFile(); |
|
229 |
// metadata.setURI(currentURI); |
|
230 |
// metadata.setObjectID("" + i); |
|
231 |
// results.add(metadata.toJSON()); |
|
232 |
// } |
|
233 |
// |
|
234 |
// } |
|
235 |
// ObjectBroker o = new ObjectBroker(Protocols.HTTP, null, null, |
|
236 |
// "jpeg"); |
|
237 |
// Iterable<ObjectStoreRecord> a = Iterables.transform(results, o); |
|
238 |
// for (ObjectStoreRecord or : a) { |
|
239 |
// |
|
240 |
// OutputStream os = new FileOutputStream( |
|
241 |
// "/home/sandro/Desktop/download/" |
|
242 |
// + or.getFileMetadata().getObjectID() + ".jpg"); |
|
243 |
// byte[] buffer = new byte[4096]; |
|
244 |
// int bytesRead; |
|
245 |
// while ((bytesRead = or.getInputStream().read(buffer)) != -1) { |
|
246 |
// os.write(buffer, 0, bytesRead); |
|
247 |
// } |
|
248 |
// or.getInputStream().close(); |
|
249 |
// os.close(); |
|
250 |
// } |
|
251 |
// |
|
252 |
// } finally { |
|
253 |
// scanner.close(); |
|
254 |
// } |
|
255 |
// |
|
256 |
// } |
|
257 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/modular/gridFS/GridFSObjectStore.java | ||
---|---|---|
1 |
package eu.dnetlib.data.objectstore.modular.gridFS; |
|
2 |
|
|
3 |
import java.io.ByteArrayInputStream; |
|
4 |
import java.io.IOException; |
|
5 |
import java.io.InputStream; |
|
6 |
import java.io.UnsupportedEncodingException; |
|
7 |
import java.net.URLEncoder; |
|
8 |
import java.util.List; |
|
9 |
import java.util.regex.Pattern; |
|
10 |
|
|
11 |
import com.mongodb.BasicDBObject; |
|
12 |
import com.mongodb.DBObject; |
|
13 |
import com.mongodb.gridfs.GridFS; |
|
14 |
import com.mongodb.gridfs.GridFSDBFile; |
|
15 |
import com.mongodb.gridfs.GridFSInputFile; |
|
16 |
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord; |
|
17 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore; |
|
18 |
import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord; |
|
19 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile; |
|
20 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException; |
|
21 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; |
|
22 |
import eu.dnetlib.enabling.resultset.ResultSetListener; |
|
23 |
import org.apache.commons.logging.Log; |
|
24 |
import org.apache.commons.logging.LogFactory; |
|
25 |
|
|
26 |
// TODO: Auto-generated Javadoc |
|
27 |
/** |
|
28 |
* The Class GridFSObjectStore. |
|
29 |
*/ |
|
30 |
@SuppressWarnings("all") |
|
31 |
public class GridFSObjectStore implements ObjectStore { |
|
32 |
|
|
33 |
/** The Constant log. */ |
|
34 |
private static final Log log = LogFactory.getLog(GridFSObjectStore.class); |
|
35 |
/** |
|
36 |
* The collection. |
|
37 |
*/ |
|
38 |
private final GridFS collection; |
|
39 |
/** The id. */ |
|
40 |
private String id; |
|
41 |
/** The upsert. */ |
|
42 |
private boolean upsert; |
|
43 |
/** The base uri. */ |
|
44 |
private String baseURI; |
|
45 |
|
|
46 |
/** |
|
47 |
* Instantiates a new grid fs object store. |
|
48 |
* |
|
49 |
* @param id |
|
50 |
* the id |
|
51 |
* @param collection |
|
52 |
* the collection |
|
53 |
* @param upsert |
|
54 |
* the upsert |
|
55 |
*/ |
|
56 |
public GridFSObjectStore(final String id, final GridFS collection, final boolean upsert) { |
|
57 |
this.id = id; |
|
58 |
this.setUpsert(upsert); |
|
59 |
this.collection = collection; |
|
60 |
|
|
61 |
} |
|
62 |
|
|
63 |
/* |
|
64 |
* (non-Javadoc) |
|
65 |
* |
|
66 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId() |
|
67 |
*/ |
|
68 |
@Override |
|
69 |
public String getId() { |
|
70 |
return id; |
|
71 |
} |
|
72 |
|
|
73 |
/* |
|
74 |
* (non-Javadoc) |
|
75 |
* |
|
76 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean) |
|
77 |
*/ |
|
78 |
@Override |
|
79 |
public int feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) { |
|
80 |
long timestamp = System.currentTimeMillis(); |
|
81 |
for (MetadataObjectRecord o : records) { |
|
82 |
if (o == null || o.getRecord() == null) { |
|
83 |
log.debug("Null object metadata record"); |
|
84 |
continue; |
|
85 |
} |
|
86 |
|
|
87 |
GridFSInputFile currentFile = collection.createFile(new ByteArrayInputStream(o.getRecord().getBytes())); |
|
88 |
currentFile.setId(o.getId()); |
|
89 |
BasicDBObject metadata = new BasicDBObject(); |
|
90 |
metadata.put("id", o.getId()); |
|
91 |
metadata.put("mime", o.getMime()); |
|
92 |
metadata.put("timestamp", timestamp); |
|
93 |
try { |
|
94 |
String URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId=" + URLEncoder.encode(o.getId(), "UTF-8"); |
|
95 |
metadata.put("uri", URI); |
|
96 |
|
|
97 |
} catch (UnsupportedEncodingException e) { |
|
98 |
log.error("Got an exception during the feed ", e); |
|
99 |
} |
|
100 |
currentFile.setMetaData(metadata); |
|
101 |
currentFile.save(); |
|
102 |
} |
|
103 |
return this.getSize(); |
|
104 |
} |
|
105 |
|
|
106 |
/* |
|
107 |
* (non-Javadoc) |
|
108 |
* |
|
109 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean) |
|
110 |
*/ |
|
111 |
@Override |
|
112 |
public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) { |
|
113 |
long timestamp = System.currentTimeMillis(); |
|
114 |
int time = 0; |
|
115 |
for (ObjectStoreRecord o : records) { |
|
116 |
if (o == null || o.getInputStream() == null) { |
|
117 |
if (o != null) { |
|
118 |
log.debug("Null object " + o.getFileMetadata().toJSON()); |
|
119 |
} else { |
|
120 |
log.debug("Null Object"); |
|
121 |
} |
|
122 |
try { |
|
123 |
Thread.sleep(1000); |
|
124 |
} catch (InterruptedException e) { |
|
125 |
log.error(e); |
|
126 |
} |
|
127 |
continue; |
|
128 |
} |
|
129 |
if (alreadyExist(o.getFileMetadata().getObjectID())) { |
|
130 |
try { |
|
131 |
o.getInputStream().close(); |
|
132 |
} catch (IOException e) { |
|
133 |
log.error("Error on closing inputStream ", e); |
|
134 |
} |
|
135 |
continue; |
|
136 |
} |
|
137 |
GridFSInputFile currentFile = collection.createFile(o.getInputStream()); |
|
138 |
currentFile.setId(o.getFileMetadata().getObjectID()); |
|
139 |
currentFile.setFilename(o.getFileMetadata().getObjectID()); |
|
140 |
BasicDBObject metadata = new BasicDBObject(); |
|
141 |
metadata.put("id", o.getFileMetadata().getObjectID()); |
|
142 |
metadata.put("mime", o.getFileMetadata().getMimeType()); |
|
143 |
metadata.put("originalObject", o.getFileMetadata().toJSON()); |
|
144 |
metadata.put("timestamp", timestamp); |
|
145 |
try { |
|
146 |
String URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId=" |
|
147 |
+ URLEncoder.encode(o.getFileMetadata().getObjectID(), "UTF-8"); |
|
148 |
metadata.put("uri", URI); |
|
149 |
|
|
150 |
} catch (UnsupportedEncodingException e) { |
|
151 |
log.error("Got an exception during the feed ", e); |
|
152 |
} |
|
153 |
currentFile.setMetaData(metadata); |
|
154 |
currentFile.save(); |
|
155 |
} |
|
156 |
return getSize(); |
|
157 |
} |
|
158 |
|
|
159 |
/* |
|
160 |
* (non-Javadoc) |
|
161 |
* |
|
162 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Double, java.lang.Double) |
|
163 |
*/ |
|
164 |
@Override |
|
165 |
public ResultSetListener deliver(final Long from, final Long until) { |
|
166 |
GridFSObjectstoreResultSetListener resulset = new GridFSObjectstoreResultSetListener(); |
|
167 |
resulset.setBaseURI(baseURI); |
|
168 |
resulset.setObjectStoreID(id); |
|
169 |
resulset.setCollection(collection); |
|
170 |
resulset.setFromDate(from); |
|
171 |
resulset.setUntilDate(until); |
|
172 |
return resulset; |
|
173 |
} |
|
174 |
|
|
175 |
/* |
|
176 |
* (non-Javadoc) |
|
177 |
* |
|
178 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable) |
|
179 |
*/ |
|
180 |
@Override |
|
181 |
public ResultSetListener deliverIds(final Iterable<String> ids) { |
|
182 |
GridFSObjectstoreResultSetListener resulset = new GridFSObjectstoreResultSetListener(); |
|
183 |
resulset.setBaseURI(baseURI); |
|
184 |
resulset.setObjectStoreID(id); |
|
185 |
resulset.setCollection(collection); |
|
186 |
resulset.setRecords((List<String>) ids); |
|
187 |
return resulset; |
|
188 |
} |
|
189 |
|
|
190 |
/* |
|
191 |
* (non-Javadoc) |
|
192 |
* |
|
193 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize() |
|
194 |
*/ |
|
195 |
@Override |
|
196 |
public int getSize() { |
|
197 |
return collection.getFileList().count(); |
|
198 |
} |
|
199 |
|
|
200 |
/* |
|
201 |
* (non-Javadoc) |
|
202 |
* |
|
203 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String) |
|
204 |
*/ |
|
205 |
@Override |
|
206 |
public void deleteObject(final String objectId) { |
|
207 |
|
|
208 |
} |
|
209 |
|
|
210 |
/* |
|
211 |
* (non-Javadoc) |
|
212 |
* |
|
213 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getObject(java.lang.String) |
|
214 |
*/ |
|
215 |
@Override |
|
216 |
public String getObject(final String recordId) { |
|
217 |
return null; |
|
218 |
} |
|
219 |
|
|
220 |
/* |
|
221 |
* (non-Javadoc) |
|
222 |
* |
|
223 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation() |
|
224 |
*/ |
|
225 |
@Override |
|
226 |
public String getInterpretation() { |
|
227 |
return (String) getMDStoreMetadata().get("interpretation"); |
|
228 |
} |
|
229 |
|
|
230 |
/** |
|
231 |
* Gets the MD store metadata. |
|
232 |
* |
|
233 |
* @return the MD store metadata |
|
234 |
*/ |
|
235 |
public DBObject getMDStoreMetadata() { |
|
236 |
return collection.getDB().getCollection("metadataObjectStore").findOne(new BasicDBObject("obsId", Pattern.compile(getId()))); |
|
237 |
} |
|
238 |
|
|
239 |
/** |
|
240 |
* Gets the base uri. |
|
241 |
* |
|
242 |
* @return the base uri |
|
243 |
*/ |
|
244 |
public String getBaseURI() { |
|
245 |
return baseURI; |
|
246 |
} |
|
247 |
|
|
248 |
/** |
|
249 |
* Sets the base uri. |
|
250 |
* |
|
251 |
* @param baseURI |
|
252 |
* the new base uri |
|
253 |
*/ |
|
254 |
public void setBaseURI(final String baseURI) { |
|
255 |
this.baseURI = baseURI; |
|
256 |
|
|
257 |
} |
|
258 |
|
|
259 |
/** |
|
260 |
* Checks if is upsert. |
|
261 |
* |
|
262 |
* @return true, if is upsert |
|
263 |
*/ |
|
264 |
public boolean isUpsert() { |
|
265 |
return upsert; |
|
266 |
} |
|
267 |
|
|
268 |
/** |
|
269 |
* Sets the upsert. |
|
270 |
* |
|
271 |
* @param upsert |
|
272 |
* the new upsert |
|
273 |
*/ |
|
274 |
public void setUpsert(final boolean upsert) { |
|
275 |
this.upsert = upsert; |
|
276 |
} |
|
277 |
|
|
278 |
/** |
|
279 |
* Already exist. |
|
280 |
* |
|
281 |
* @param objectId |
|
282 |
* the object id |
|
283 |
* @return true, if successful |
|
284 |
*/ |
|
285 |
private boolean alreadyExist(final String objectId) { |
|
286 |
BasicDBObject query = new BasicDBObject("_id", objectId); |
|
287 |
List<GridFSDBFile> file = collection.find(query); |
|
288 |
return file != null && file.size() > 0; |
|
289 |
} |
|
290 |
|
|
291 |
/* |
|
292 |
* (non-Javadoc) |
|
293 |
* |
|
294 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String) |
|
295 |
*/ |
|
296 |
@Override |
|
297 |
public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException { |
|
298 |
BasicDBObject query = new BasicDBObject("_id", objectId); |
|
299 |
List<GridFSDBFile> file = collection.find(query); |
|
300 |
|
|
301 |
checkSingleItem(objectId, file); |
|
302 |
|
|
303 |
return ObjectStoreFileUtility.build(file.get(0), baseURI, id); |
|
304 |
|
|
305 |
} |
|
306 |
|
|
307 |
/** |
|
308 |
* Check single item. |
|
309 |
* |
|
310 |
* @param objectId |
|
311 |
* the object id |
|
312 |
* @param file |
|
313 |
* the file |
|
314 |
* @throws ObjectStoreFileNotFoundException |
|
315 |
* the object store file not found exception |
|
316 |
*/ |
|
317 |
private void checkSingleItem(final String objectId, final List<GridFSDBFile> file) throws ObjectStoreFileNotFoundException { |
|
318 |
if (file.isEmpty()) { throw new ObjectStoreFileNotFoundException(String.format("Object file not found with id: %s", objectId)); } |
|
319 |
if (file.size() > 1) { throw new IllegalStateException(String.format("More than one objects found with id: %s", objectId)); } |
|
320 |
} |
|
321 |
|
|
322 |
/* |
|
323 |
* (non-Javadoc) |
|
324 |
* |
|
325 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverStream(java.lang.String) |
|
326 |
*/ |
|
327 |
@Override |
|
328 |
public InputStream deliverStream(final String objectId) throws ObjectStoreFileNotFoundException { |
|
329 |
BasicDBObject query = new BasicDBObject("_id", objectId); |
|
330 |
List<GridFSDBFile> file = collection.find(query); |
|
331 |
checkSingleItem(objectId, file); |
|
332 |
return file.get(0).getInputStream(); |
|
333 |
} |
|
334 |
|
|
335 |
/* |
|
336 |
* (non-Javadoc) |
|
337 |
* |
|
338 |
* @see |
|
339 |
* eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord) |
|
340 |
*/ |
|
341 |
@Override |
|
342 |
public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException { |
|
343 |
if (record == null || record.getFileMetadata() == null) { throw new ObjectStoreServiceException("Empty input Record"); } |
|
344 |
|
|
345 |
if (existIDStartsWith(record.getFileMetadata().getObjectID())) { |
|
346 |
log.debug("Object already exist "); |
|
347 |
if (record.getInputStream() != null) { |
|
348 |
try { |
|
349 |
record.getInputStream().close(); |
|
350 |
} catch (IOException e) { |
|
351 |
log.debug("Exception happen in closing inputstream " + e); |
|
352 |
throw new ObjectStoreServiceException(e); |
|
353 |
} |
|
354 |
} |
|
355 |
ObjectStoreFile obj = deliverObject(record.getFileMetadata().getObjectID()); |
|
356 |
return obj.getURI(); |
|
357 |
} |
|
358 |
long timestamp = System.currentTimeMillis(); |
|
359 |
String URI = ""; |
|
360 |
|
|
361 |
if (record.getInputStream() == null) { throw new ObjectStoreServiceException("missing inputstream on record " + record.getFileMetadata().getObjectID()); } |
|
362 |
GridFSInputFile currentFile = collection.createFile(record.getInputStream()); |
|
363 |
currentFile.setId(record.getFileMetadata().getObjectID()); |
|
364 |
currentFile.setFilename(record.getFileMetadata().getObjectID()); |
|
365 |
BasicDBObject metadata = new BasicDBObject(); |
|
366 |
metadata.put("id", record.getFileMetadata().getObjectID()); |
|
367 |
metadata.put("mime", record.getFileMetadata().getMimeType()); |
|
368 |
metadata.put("originalObject", record.getFileMetadata().toJSON()); |
|
369 |
|
|
370 |
metadata.put("timestamp", timestamp); |
|
371 |
try { |
|
372 |
URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId=" |
|
373 |
+ URLEncoder.encode(record.getFileMetadata().getObjectID(), "UTF-8"); |
|
374 |
metadata.put("uri", URI); |
|
375 |
currentFile.setMetaData(metadata); |
|
376 |
currentFile.save(); |
|
377 |
|
|
378 |
} catch (Exception e) { |
|
379 |
log.error(e); |
|
380 |
throw new ObjectStoreServiceException(e); |
|
381 |
} finally { |
|
382 |
try { |
|
383 |
record.getInputStream().close(); |
|
384 |
} catch (IOException e) { |
|
385 |
log.error(e); |
|
386 |
} |
|
387 |
} |
|
388 |
return URI; |
|
389 |
} |
|
390 |
|
|
391 |
/* |
|
392 |
* (non-Javadoc) |
|
393 |
* |
|
394 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String) |
|
395 |
*/ |
|
396 |
@Override |
|
397 |
public boolean existIDStartsWith(final String startId) { |
|
398 |
BasicDBObject query = new BasicDBObject("_id", startId); |
|
399 |
List<GridFSDBFile> file = collection.find(query); |
|
400 |
return file.size() > 0; |
|
401 |
} |
|
402 |
|
|
403 |
@Override |
|
404 |
public boolean dropContent() throws ObjectStoreServiceException { |
|
405 |
this.collection.remove(new BasicDBObject()); |
|
406 |
return true; |
|
407 |
} |
|
408 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/modular/gridFS/MongoGridFSOptionsFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.data.objectstore.modular.gridFS; |
|
2 |
|
|
3 |
import org.springframework.beans.BeansException; |
|
4 |
import org.springframework.beans.factory.FactoryBean; |
|
5 |
|
|
6 |
import com.mongodb.MongoOptions; |
|
7 |
|
|
8 |
public class MongoGridFSOptionsFactory implements FactoryBean<MongoOptions> { |
|
9 |
|
|
10 |
private int connectionsPerHost; |
|
11 |
|
|
12 |
@Override |
|
13 |
public MongoOptions getObject() throws BeansException { |
|
14 |
MongoOptions opts = new MongoOptions(); |
|
15 |
opts.connectionsPerHost = connectionsPerHost; |
|
16 |
return opts; |
|
17 |
} |
|
18 |
|
|
19 |
@Override |
|
20 |
public Class<MongoOptions> getObjectType() { |
|
21 |
return MongoOptions.class; |
|
22 |
} |
|
23 |
|
|
24 |
@Override |
|
25 |
public boolean isSingleton() { |
|
26 |
return false; |
|
27 |
} |
|
28 |
|
|
29 |
public int getConnectionsPerHost() { |
|
30 |
return connectionsPerHost; |
|
31 |
} |
|
32 |
|
|
33 |
public void setConnectionsPerHost(final int connectionsPerHost) { |
|
34 |
this.connectionsPerHost = connectionsPerHost; |
|
35 |
} |
|
36 |
|
|
37 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/modular/gridFS/GridFSObjectstoreResultSetListener.java | ||
---|---|---|
1 |
package eu.dnetlib.data.objectstore.modular.gridFS; |
|
2 |
|
|
3 |
import java.util.ArrayList; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
import org.apache.commons.logging.Log; |
|
7 |
import org.apache.commons.logging.LogFactory; |
|
8 |
|
|
9 |
import com.google.common.collect.Lists; |
|
10 |
import com.mongodb.BasicDBObject; |
|
11 |
import com.mongodb.DBCursor; |
|
12 |
import com.mongodb.DBObject; |
|
13 |
import com.mongodb.QueryBuilder; |
|
14 |
import com.mongodb.gridfs.GridFS; |
|
15 |
import com.mongodb.gridfs.GridFSDBFile; |
|
16 |
|
|
17 |
import eu.dnetlib.enabling.resultset.ResultSet; |
|
18 |
import eu.dnetlib.enabling.resultset.ResultSetAware; |
|
19 |
import eu.dnetlib.enabling.resultset.ResultSetListener; |
|
20 |
import eu.dnetlib.miscutils.collections.MappedCollection; |
|
21 |
|
|
22 |
/** |
|
23 |
* The listener interface for receiving gridFSObjectstoreResultSet events. The class that is interested in processing a |
|
24 |
* gridFSObjectstoreResultSet event implements this interface, and the object created with that class is registered with a component using |
|
25 |
* the component's <code>addGridFSObjectstoreResultSetListener<code> method. When |
|
26 |
* the gridFSObjectstoreResultSet event occurs, that object's appropriate |
|
27 |
* method is invoked. |
|
28 |
* |
|
29 |
* @see GridFSObjectstoreResultSetEvent |
|
30 |
*/ |
|
31 |
public class GridFSObjectstoreResultSetListener implements ResultSetListener, ResultSetAware { |
|
32 |
|
|
33 |
/** The Constant log. */ |
|
34 |
private static final Log log = LogFactory.getLog(GridFSObjectstoreResultSetListener.class); |
|
35 |
|
|
36 |
/** The from date. */ |
|
37 |
private Long fromDate; |
|
38 |
|
|
39 |
/** The until date. */ |
|
40 |
private Long untilDate; |
|
41 |
|
|
42 |
/** The records. */ |
|
43 |
private List<String> records; |
|
44 |
|
|
45 |
/** The object store id. */ |
|
46 |
private String objectStoreID; |
|
47 |
|
|
48 |
/** The collection. */ |
|
49 |
private GridFS collection; |
|
50 |
|
|
51 |
/** The base uri. */ |
|
52 |
private String baseURI; |
|
53 |
|
|
54 |
/** The current size. */ |
|
55 |
private int currentSize = -1; |
|
56 |
|
|
57 |
/** The current cursor. */ |
|
58 |
private DBCursor currentCursor; |
|
59 |
|
|
60 |
/** The cursor position. */ |
|
61 |
private long cursorPosition; |
|
62 |
|
|
63 |
/* |
|
64 |
* (non-Javadoc) |
|
65 |
* |
|
66 |
* @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int) |
|
67 |
*/ |
|
68 |
@Override |
|
69 |
public List<String> getResult(final int from, final int to) { |
|
70 |
log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to)); |
|
71 |
if (records != null) { |
|
72 |
List<String> ids = Lists.newArrayList(); |
|
73 |
for (int i = from; i < to; i++) { |
|
74 |
ids.add(records.get(i)); |
|
75 |
} |
|
76 |
QueryBuilder qBuilder = QueryBuilder.start("metadata.id").in(ids); |
|
77 |
DBObject q = qBuilder.get(); |
|
78 |
List<GridFSDBFile> out = collection.find(q); |
|
79 |
|
|
80 |
return MappedCollection.listMap(out, ObjectStoreFileUtility.asJSON(baseURI, objectStoreID)); |
|
81 |
} else if ((fromDate != null) && (untilDate != null)) { |
|
82 |
if ((currentCursor == null) || (cursorPosition > from)) { |
|
83 |
createCurrentCursor(); |
|
84 |
} |
|
85 |
while (cursorPosition < from) { |
|
86 |
currentCursor.next(); |
|
87 |
cursorPosition++; |
|
88 |
} |
|
89 |
ArrayList<GridFSDBFile> out = new ArrayList<GridFSDBFile>(); |
|
90 |
|
|
91 |
for (int i = from; i <= to; i++) { |
|
92 |
if (currentCursor.hasNext()) { |
|
93 |
out.add((GridFSDBFile) currentCursor.next()); |
|
94 |
cursorPosition++; |
|
95 |
} |
|
96 |
} |
|
97 |
|
|
98 |
return MappedCollection.listMap(out, ObjectStoreFileUtility.asJSON(baseURI, objectStoreID)); |
|
99 |
} |
|
100 |
|
|
101 |
throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs"); |
|
102 |
} |
|
103 |
|
|
104 |
/** |
|
105 |
* Creates the current cursor. |
|
106 |
*/ |
|
107 |
private void createCurrentCursor() { |
|
108 |
BasicDBObject query = new BasicDBObject(); |
|
109 |
query.put("$gt", fromDate.doubleValue()); |
|
110 |
query.put("$lt", untilDate.doubleValue()); |
|
111 |
if (currentCursor != null) { |
|
112 |
currentCursor.close(); |
|
113 |
} |
|
114 |
currentCursor = collection.getFileList(new BasicDBObject("metadata.timestamp", query)).sort(new BasicDBObject("_id", 1)); |
|
115 |
cursorPosition = 1; |
|
116 |
|
|
117 |
} |
|
118 |
|
|
119 |
/* |
|
120 |
* (non-Javadoc) |
|
121 |
* |
|
122 |
* @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize() |
|
123 |
*/ |
|
124 |
@Override |
|
125 |
public int getSize() { |
|
126 |
if (currentSize == -1) { |
|
127 |
currentSize = calculateSize(); |
|
128 |
} |
|
129 |
return currentSize - 1; |
|
130 |
} |
|
131 |
|
|
132 |
/** |
|
133 |
* Calculate size. |
|
134 |
* |
|
135 |
* @return the int |
|
136 |
*/ |
|
137 |
private int calculateSize() { |
|
138 |
if (records != null) { |
|
139 |
QueryBuilder qBuilder = QueryBuilder.start("metadata.id").in(records); |
|
140 |
DBObject q = qBuilder.get(); |
|
141 |
List<GridFSDBFile> out = collection.find(q); |
|
142 |
return out.size(); |
|
143 |
} else if ((fromDate != null) && (untilDate != null)) { |
|
144 |
BasicDBObject query = new BasicDBObject(); |
|
145 |
query.put("$gt", fromDate.doubleValue()); |
|
146 |
query.put("$lt", untilDate.doubleValue()); |
|
147 |
return collection.getFileList(new BasicDBObject("metadata.timestamp", query)).size(); |
|
148 |
} |
|
149 |
return 0; |
|
150 |
} |
|
151 |
|
|
152 |
/* |
|
153 |
* (non-Javadoc) |
|
154 |
* |
|
155 |
* @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet) |
|
156 |
*/ |
|
157 |
@Override |
|
158 |
public void setResultSet(final ResultSet resultSet) { |
|
159 |
resultSet.close(); |
|
160 |
} |
|
161 |
|
|
162 |
/** |
|
163 |
* Gets the from date. |
|
164 |
* |
|
165 |
* @return the from date |
|
166 |
*/ |
|
167 |
public Long getFromDate() { |
|
168 |
return fromDate; |
|
169 |
} |
|
170 |
|
|
171 |
/** |
|
172 |
* Sets the from date. |
|
173 |
* |
|
174 |
* @param fromdate |
|
175 |
* the new from date |
|
176 |
*/ |
|
177 |
public void setFromDate(final Long fromdate) { |
|
178 |
this.fromDate = fromdate; |
|
179 |
} |
|
180 |
|
|
181 |
/** |
|
182 |
* Gets the until date. |
|
183 |
* |
|
184 |
* @return the until date |
|
185 |
*/ |
|
186 |
public Long getUntilDate() { |
|
187 |
return untilDate; |
|
188 |
} |
|
189 |
|
|
190 |
/** |
|
191 |
* Sets the until date. |
|
192 |
* |
|
193 |
* @param untilDate |
|
194 |
* the new until date |
|
195 |
*/ |
|
196 |
public void setUntilDate(final Long untilDate) { |
|
197 |
this.untilDate = untilDate; |
|
198 |
} |
|
199 |
|
|
200 |
/** |
|
201 |
* Gets the records. |
|
202 |
* |
|
203 |
* @return the records |
|
204 |
*/ |
|
205 |
public List<String> getRecords() { |
|
206 |
return records; |
|
207 |
} |
|
208 |
|
|
209 |
/** |
|
210 |
* Sets the records. |
|
211 |
* |
|
212 |
* @param records |
|
213 |
* the new records |
|
214 |
*/ |
|
215 |
public void setRecords(final List<String> records) { |
|
216 |
this.records = records; |
|
217 |
} |
|
218 |
|
|
219 |
/** |
|
220 |
* Gets the collection. |
|
221 |
* |
|
222 |
* @return the collection |
|
223 |
*/ |
|
224 |
public GridFS getCollection() { |
|
225 |
return collection; |
|
226 |
} |
|
227 |
|
|
228 |
/** |
|
229 |
* Sets the collection. |
|
230 |
* |
|
231 |
* @param collection |
|
232 |
* the new collection |
|
233 |
*/ |
|
234 |
public void setCollection(final GridFS collection) { |
|
235 |
this.collection = collection; |
|
236 |
} |
|
237 |
|
|
238 |
/** |
|
239 |
* Gets the base uri. |
|
240 |
* |
|
241 |
* @return the base uri |
|
242 |
*/ |
|
243 |
public String getBaseURI() { |
|
244 |
return baseURI; |
|
245 |
} |
|
246 |
|
|
247 |
/** |
|
248 |
* Sets the base uri. |
|
249 |
* |
|
250 |
* @param baseURI |
|
251 |
* the new base uri |
|
252 |
*/ |
|
253 |
public void setBaseURI(final String baseURI) { |
|
254 |
this.baseURI = baseURI; |
|
255 |
} |
|
256 |
|
|
257 |
/** |
|
258 |
* Gets the object store id. |
|
259 |
* |
|
260 |
* @return the object store id |
|
261 |
*/ |
|
262 |
public String getObjectStoreID() { |
|
263 |
return objectStoreID; |
|
264 |
} |
|
265 |
|
|
266 |
/** |
|
267 |
* Sets the object store id. |
|
268 |
* |
|
269 |
* @param objectStoreID |
|
270 |
* the new object store id |
|
271 |
*/ |
|
272 |
public void setObjectStoreID(final String objectStoreID) { |
|
273 |
this.objectStoreID = objectStoreID; |
|
274 |
} |
|
275 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/modular/gridFS/ObjectStoreFileUtility.java | ||
---|---|---|
1 |
package eu.dnetlib.data.objectstore.modular.gridFS; |
|
2 |
|
|
3 |
import java.io.UnsupportedEncodingException; |
|
4 |
import java.net.URLEncoder; |
|
5 |
|
|
6 |
import org.apache.commons.logging.Log; |
|
7 |
import org.apache.commons.logging.LogFactory; |
|
8 |
|
|
9 |
import com.mongodb.DBObject; |
|
10 |
import com.mongodb.gridfs.GridFSDBFile; |
|
11 |
|
|
12 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile; |
|
13 |
import eu.dnetlib.data.objectstore.rmi.Protocols; |
|
14 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
15 |
|
|
16 |
/** |
|
17 |
* The Class ObjectStoreFileBuilder generates an objectStoreFile bean |
|
18 |
* |
|
19 |
*/ |
|
20 |
public class ObjectStoreFileUtility { |
|
21 |
|
|
22 |
private static final int KB_SIZE = 1024; |
|
23 |
|
|
24 |
/** The Constant log. */ |
|
25 |
private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class); |
|
26 |
|
|
27 |
/** |
|
28 |
* Builds the. |
|
29 |
* |
|
30 |
* @param input |
|
31 |
* the input File |
|
32 |
* @param baseURI |
|
33 |
* the base URI to associate to the metadata |
|
34 |
* @param objectStoreID |
|
35 |
* The objectStoreID the input file |
|
36 |
* @return the object store file |
|
37 |
*/ |
|
38 |
|
|
39 |
public static ObjectStoreFile build(final GridFSDBFile input, final String baseURI, final String objectStoreID) { |
|
40 |
DBObject metadata = input.getMetaData(); |
|
41 |
String originalFile = (String) metadata.get("originalObject"); |
|
42 |
ObjectStoreFile original = ObjectStoreFile.createObject(originalFile); |
|
43 |
ObjectStoreFile newFile = new ObjectStoreFile(); |
|
44 |
newFile.setObjectID((String) metadata.get("id")); |
|
45 |
newFile.setAccessProtocol(Protocols.HTTP); |
|
46 |
newFile.setMimeType((String) metadata.get("mime")); |
|
47 |
newFile.setMd5Sum((String) input.get("md5")); |
|
48 |
newFile.setFileSizeKB(input.getLength() / KB_SIZE); |
|
49 |
if (originalFile != null) { |
|
50 |
newFile.setMetadataRelatedID(original.getMetadataRelatedID()); |
|
51 |
if (original.getDownloadedURL() == null || original.getDownloadedURL().length() == 0) { |
|
52 |
newFile.setDownloadedURL(original.getURI()); |
|
53 |
} else { |
|
54 |
newFile.setDownloadedURL(original.getDownloadedURL()); |
|
55 |
} |
|
56 |
|
|
57 |
} |
|
58 |
try { |
|
59 |
newFile.setURI(baseURI + "?objectStore=" + URLEncoder.encode(objectStoreID, "UTF-8") + "&objectId=" |
|
60 |
+ URLEncoder.encode(newFile.getObjectID(), "UTF-8")); |
|
61 |
} catch (UnsupportedEncodingException e) { |
|
62 |
log.error("Error on Build objectStoreFile ", e); |
|
63 |
} |
|
64 |
return newFile; |
|
65 |
} |
|
66 |
|
|
67 |
public static UnaryFunction<String, GridFSDBFile> asJSON(final String baseURI, final String objectStoreID) { |
|
68 |
return new UnaryFunction<String, GridFSDBFile>() { |
|
69 |
|
|
70 |
@Override |
|
71 |
public String evaluate(final GridFSDBFile input) { |
|
72 |
return build(input, baseURI, objectStoreID).toJSON(); |
|
73 |
|
|
74 |
} |
|
75 |
}; |
|
76 |
} |
|
77 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/modular/gridFS/GridFSObjectstoreDaoImpl.java | ||
---|---|---|
1 |
package eu.dnetlib.data.objectstore.modular.gridFS; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.regex.Pattern; |
|
5 |
|
|
6 |
import com.mongodb.*; |
|
7 |
import com.mongodb.gridfs.GridFS; |
|
8 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore; |
|
9 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao; |
|
10 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; |
|
11 |
import eu.dnetlib.miscutils.collections.MappedCollection; |
|
12 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
13 |
import org.bson.BSONObject; |
|
14 |
import org.springframework.beans.factory.annotation.Required; |
|
15 |
|
|
16 |
// TODO: Auto-generated Javadoc |
|
17 |
/** |
|
18 |
* The Class GridFSObjectstoreDaoImpl. |
|
19 |
*/ |
|
20 |
@SuppressWarnings("all") |
|
21 |
public class GridFSObjectstoreDaoImpl implements ObjectStoreDao { |
|
22 |
|
|
23 |
/** The Constant INTERPRETATION. */ |
|
24 |
public static final String INTERPRETATION = "interpretation"; |
|
25 |
|
|
26 |
/** The Constant OBJECTSTOREMETADATANAME. */ |
|
27 |
private final static String OBJECTSTOREMETADATANAME = "metadataObjectStore"; |
|
28 |
|
|
29 |
/** The Constant OBJECTSTOREIDFIELD. */ |
|
30 |
private final static String OBJECTSTOREIDFIELD = "obsId"; |
|
31 |
|
|
32 |
/** The db. */ |
|
33 |
private DB db; |
|
34 |
|
|
35 |
/** The upsert. */ |
|
36 |
private boolean upsert; |
|
37 |
|
|
38 |
/** The object store resturi. */ |
|
39 |
private String objectStoreRESTURI; |
|
40 |
|
|
41 |
/** |
|
42 |
* {@inheritDoc} |
|
43 |
* |
|
44 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#getObjectStore(java.lang.String) |
|
45 |
*/ |
|
46 |
@Override |
|
47 |
public ObjectStore getObjectStore(String obsId) { |
|
48 |
db.setWriteConcern(WriteConcern.SAFE); |
|
49 |
obsId = obsId.substring(0, 36); |
|
50 |
GridFSObjectStore objectStore = new GridFSObjectStore(obsId, new GridFS(db, obsId), this.upsert); |
|
51 |
objectStore.setBaseURI(objectStoreRESTURI); |
|
52 |
checkIndexes(obsId); |
|
53 |
return objectStore; |
|
54 |
} |
|
55 |
|
|
56 |
/** |
|
57 |
* Check indexes. |
|
58 |
* |
|
59 |
* @param id |
|
60 |
* the id |
|
61 |
*/ |
|
62 |
private void checkIndexes(final String id) { |
|
63 |
DBCollection coll = db.getCollection(id + ".files"); |
|
64 |
List<DBObject> indexInfo = coll.getIndexInfo(); |
|
65 |
boolean hasTimestampIndex = false; |
|
66 |
for (DBObject info : indexInfo) { |
|
67 |
BSONObject boj = (BSONObject) info.get("key"); |
|
68 |
if (boj.get("metadata.timestamp") != null) { |
|
69 |
hasTimestampIndex = true; |
|
70 |
} |
|
71 |
} |
|
72 |
if (!hasTimestampIndex) { |
|
73 |
coll.createIndex("metadata.timestamp"); |
|
74 |
} |
|
75 |
} |
|
76 |
|
|
77 |
/** |
|
78 |
* {@inheritDoc} |
|
79 |
* |
|
80 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#listObjectStores() |
|
81 |
*/ |
|
82 |
@Override |
|
83 |
public List<String> listObjectStores() { |
|
84 |
DBCollection metadata = db.getCollection(OBJECTSTOREMETADATANAME); |
|
85 |
return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() { |
|
86 |
|
|
87 |
@Override |
|
88 |
public String evaluate(final DBObject object) { |
|
89 |
return (String) object.get(OBJECTSTOREIDFIELD); |
|
90 |
} |
|
91 |
}); |
|
92 |
|
|
93 |
} |
|
94 |
|
|
95 |
/** |
|
96 |
* {@inheritDoc} |
|
97 |
* |
|
98 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#createObjectStore(java.lang.String, java.lang.String, |
|
99 |
* java.lang.String) |
|
100 |
*/ |
|
101 |
@Override |
|
102 |
public boolean createObjectStore(final String obsId, final String interpretation, final String basePath) throws ObjectStoreServiceException { |
|
103 |
try { |
|
104 |
DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME); |
|
105 |
final BasicDBObject obj = new BasicDBObject(); |
|
106 |
obj.put(OBJECTSTOREIDFIELD, obsId); |
|
107 |
obj.put(INTERPRETATION, interpretation); |
|
108 |
coll.save(obj); |
|
109 |
} catch (Throwable e) { |
|
110 |
throw new ObjectStoreServiceException("Error on create object store with id: " + obsId, e); |
|
111 |
} |
|
112 |
return true; |
|
113 |
} |
|
114 |
|
|
115 |
/** |
|
116 |
* {@inheritDoc} |
|
117 |
* |
|
118 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#updateObjectStore(java.lang.String, java.lang.String) |
|
119 |
*/ |
|
120 |
@Override |
|
121 |
public boolean updateObjectStore(final String obsId, final String interpretation) { |
|
122 |
DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME); |
|
123 |
final BasicDBObject obj = new BasicDBObject(); |
|
124 |
obj.put(OBJECTSTOREIDFIELD, obsId); |
|
125 |
obj.put(INTERPRETATION, interpretation); |
|
126 |
coll.update(new BasicDBObject(OBJECTSTOREIDFIELD, obsId), obj, true, false); |
|
127 |
coll.save(obj); |
|
128 |
return true; |
|
129 |
} |
|
130 |
|
|
131 |
/** |
|
132 |
* {@inheritDoc} |
|
133 |
* |
|
134 |
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#deleteObjectStore(java.lang.String) |
|
135 |
*/ |
|
136 |
@Override |
|
137 |
public boolean deleteObjectStore(String obsId) { |
|
138 |
DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME); |
|
139 |
coll.remove(new BasicDBObject(OBJECTSTOREIDFIELD, obsId)); |
|
140 |
obsId = obsId.substring(0, 36); |
|
141 |
GridFS objectStore = new GridFS(db, obsId); |
|
142 |
Pattern any = Pattern.compile("."); |
|
143 |
BasicDBObject query = new BasicDBObject("md5", any); |
|
144 |
objectStore.remove(query); |
|
145 |
return true; |
|
146 |
} |
|
147 |
|
|
148 |
@Override |
|
149 |
public boolean dropContent(final String obsId) throws ObjectStoreServiceException { |
|
150 |
return getObjectStore(obsId).dropContent(); |
|
151 |
} |
|
152 |
|
|
153 |
/** |
|
154 |
* Gets the db. |
|
155 |
* |
|
156 |
* @return the db |
|
157 |
*/ |
|
158 |
public DB getDb() { |
|
159 |
return db; |
|
160 |
} |
|
161 |
|
|
162 |
/** |
|
163 |
* Sets the db. |
|
164 |
* |
|
165 |
* @param db |
|
166 |
* the new db |
|
167 |
*/ |
|
168 |
@Required |
|
169 |
public void setDb(final DB db) { |
|
170 |
this.db = db; |
|
171 |
} |
|
172 |
|
|
173 |
/** |
|
174 |
* Checks if is upsert. |
|
175 |
* |
|
176 |
* @return true, if is upsert |
|
177 |
*/ |
|
178 |
public boolean isUpsert() { |
|
179 |
return upsert; |
|
180 |
} |
|
181 |
|
|
182 |
/** |
|
183 |
* Sets the upsert. |
|
184 |
* |
|
185 |
* @param upsert |
|
186 |
* the new upsert |
|
187 |
*/ |
|
188 |
@Required |
|
189 |
public void setUpsert(final boolean upsert) { |
|
190 |
this.upsert = upsert; |
|
191 |
} |
|
192 |
|
|
193 |
/** |
|
194 |
* Gets the object store resturi. |
|
195 |
* |
|
196 |
* @return the object store resturi |
|
197 |
*/ |
|
198 |
public String getObjectStoreRESTURI() { |
|
199 |
return objectStoreRESTURI; |
|
200 |
} |
|
201 |
|
|
202 |
/** |
|
203 |
* Sets the object store resturi. |
|
204 |
* |
|
205 |
* @param objectStoreRESTURI |
|
206 |
* the new object store resturi |
|
207 |
*/ |
|
208 |
@Required |
|
209 |
public void setObjectStoreRESTURI(final String objectStoreRESTURI) { |
|
210 |
this.objectStoreRESTURI = objectStoreRESTURI; |
|
211 |
} |
|
212 |
|
|
213 |
} |
modules/dnet-gridfs-objectstore/trunk/src/main/resources/eu/dnetlib/data/objectstore/modular/gridFS/applicationContext-gridFS-objectstore.properties | ||
---|---|---|
1 |
services.objectStore.mongodb.host=localhost |
|
2 |
services.objectStore.mongodb.port=27017 |
|
3 |
services.objectStore.mongodb.db=objectStore |
|
4 |
services.objectStore.mongodb.connectionsPerHost=20 |
|
5 |
services.objectStore.mongodb.upsert=true |
|
6 |
services.objectStore.RESTURI=http://${container.hostname}:${container.port}/${container.context}/mvc/objectStore/retrieve.do |
modules/dnet-gridfs-objectstore/trunk/src/main/resources/eu/dnetlib/data/objectstore/modular/gridFS/applicationContext-gridFS-objectstore.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<beans xmlns="http://www.springframework.org/schema/beans" |
|
3 |
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p" |
|
4 |
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> |
|
5 |
<bean id="gridFSObjectstoreDao" |
|
6 |
class="eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl" |
|
7 |
p:db-ref="objectstoreMongoDB" |
|
8 |
p:objectStoreRESTURI-ref="restURIString" |
|
9 |
p:upsert="${services.objectStore.mongodb.upsert}" |
|
10 |
/> |
|
11 |
|
|
12 |
<bean id="restURIString" class="java.lang.String"> |
|
13 |
<constructor-arg value="${services.objectStore.RESTURI}"/> |
|
14 |
</bean> |
|
15 |
|
|
16 |
<bean id="objectStoreMongoServer" class="com.mongodb.Mongo"> |
|
17 |
<constructor-arg index="0" type="com.mongodb.ServerAddress"> |
|
18 |
<bean class="com.mongodb.ServerAddress"> |
|
19 |
<constructor-arg index="0" |
|
20 |
value="${services.objectStore.mongodb.host}" /> |
|
21 |
<constructor-arg index="1" |
|
22 |
value="${services.objectStore.mongodb.port}" /> |
|
23 |
</bean> |
|
24 |
</constructor-arg> |
|
25 |
<constructor-arg index="1" type="com.mongodb.MongoOptions"> |
|
26 |
<bean class="eu.dnetlib.data.objectstore.modular.gridFS.MongoGridFSOptionsFactory" |
|
27 |
p:connectionsPerHost="${services.objectStore.mongodb.connectionsPerHost}" |
|
28 |
/> |
|
29 |
</constructor-arg> |
|
30 |
</bean> |
|
31 |
|
|
32 |
<bean id="objectstoreMongoDB" factory-bean="objectStoreMongoServer" |
|
33 |
factory-method="getDB"> |
|
34 |
<constructor-arg index="0" |
|
35 |
value="${services.objectStore.mongodb.db}" /> |
|
36 |
</bean> |
|
37 |
|
|
38 |
|
|
39 |
</beans> |
modules/dnet-gridfs-objectstore/trunk/pom.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
3 |
<parent> |
|
4 |
<groupId>eu.dnetlib</groupId> |
|
5 |
<artifactId>dnet-parent</artifactId> |
|
6 |
<version>1.0.0</version> |
|
7 |
</parent> |
|
8 |
<modelVersion>4.0.0</modelVersion> |
|
9 |
<groupId>eu.dnetlib</groupId> |
|
10 |
<artifactId>dnet-gridfs-objectstore</artifactId> |
|
11 |
<packaging>jar</packaging> |
|
12 |
<version>4.1.1-SNAPSHOT</version> |
|
13 |
<scm> |
|
14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-gridfs-objectstore/trunk</developerConnection> |
|
15 |
</scm> |
|
16 |
<dependencies> |
|
17 |
<dependency> |
|
18 |
<groupId>eu.dnetlib</groupId> |
|
19 |
<artifactId>dnet-modular-objectstore-service</artifactId> |
|
20 |
<version>[4.0.0,5.0.0)</version> |
|
21 |
</dependency> |
|
22 |
<dependency> |
|
23 |
<groupId>eu.dnetlib</groupId> |
|
24 |
<artifactId>cnr-resultset-service</artifactId> |
|
25 |
<version>[2.0.0,3.0.0)</version> |
|
26 |
</dependency> |
|
27 |
<dependency> |
|
28 |
<groupId>eu.dnetlib</groupId> |
|
29 |
<artifactId>cnr-resultset-client</artifactId> |
|
30 |
<version>[2.0.0,3.0.0)</version> |
|
31 |
</dependency> |
|
32 |
<dependency> |
|
33 |
<groupId>junit</groupId> |
|
34 |
<artifactId>junit</artifactId> |
|
35 |
<version>${junit.version}</version> |
|
36 |
<scope>test</scope> |
|
37 |
</dependency> |
|
38 |
<dependency> |
|
39 |
<groupId>org.mongodb</groupId> |
|
40 |
<artifactId>mongo-java-driver</artifactId> |
|
41 |
<version>${mongodb.driver.version}</version> |
|
42 |
</dependency> |
|
43 |
</dependencies> |
|
44 |
</project> |
modules/dnet-gridfs-objectstore/branches/tag/deploy.info | ||
---|---|---|
1 |
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-gridfs-objectstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-gridfs-objectstore"} |
modules/dnet-gridfs-objectstore/branches/tag/src/test/java/eu/dnetlib/test/TestQuery.java | ||
---|---|---|
1 |
package eu.dnetlib.test; |
|
2 |
|
|
3 |
import com.google.common.collect.Lists; |
|
4 |
import com.mongodb.*; |
|
5 |
import com.mongodb.gridfs.GridFS; |
|
6 |
import eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl; |
|
7 |
import org.junit.Ignore; |
|
8 |
import org.junit.Test; |
|
9 |
|
|
10 |
import java.io.UnsupportedEncodingException; |
|
11 |
import java.net.URLEncoder; |
|
12 |
import java.net.UnknownHostException; |
|
13 |
import java.util.ArrayList; |
|
14 |
import java.util.Date; |
|
15 |
|
|
16 |
public class TestQuery { |
|
17 |
|
Also available in: Unified diff
codebase used to migrate to java8 the production system