Project

General

Profile

« Previous | Next » 

Revision 48019

[maven-release-plugin] copy for tag dnet-fs-objectstore-2.1.8

View differences:

modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-fs-objectstore/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-fs-objectstore"}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import java.io.File;
4
import java.io.FileNotFoundException;
5
import java.io.IOException;
6
import java.nio.file.FileSystems;
7
import java.nio.file.Files;
8
import java.nio.file.Path;
9
import java.util.List;
10

  
11
import com.mongodb.client.MongoDatabase;
12
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
13
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
14
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
15
import eu.dnetlib.enabling.resultset.ResultSetListener;
16
import org.apache.commons.io.FileUtils;
17
import org.apache.commons.logging.Log;
18
import org.apache.commons.logging.LogFactory;
19
import org.junit.After;
20
import org.junit.Before;
21
import org.junit.Ignore;
22
import org.junit.Test;
23
import org.junit.runner.RunWith;
24
import org.springframework.beans.factory.annotation.Autowired;
25
import org.springframework.test.context.ContextConfiguration;
26
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
27

  
28
import static org.junit.Assert.assertNotNull;
29
import static org.junit.Assert.assertTrue;
30

  
31
/**
32
 * TODO: Test class set to ignored because it requires a mongo server running on localhost. Tests with mocks should be prepared.
33
 * @author sandro
34
 *
35
 */
36
@Ignore
37
@RunWith(SpringJUnit4ClassRunner.class)
38
@ContextConfiguration(classes = ConfigurationTestConfig.class)
39
public class ObjectStoreServiceDAOTest {
40
	private static final Log log = LogFactory.getLog(ObjectStoreServiceDAOTest.class); // NOPMD by marko on 11/24/08 5:02 PM
41

  
42
	@Autowired
43
	private FileSystemObjectStoreDao objectStoreDAO;
44

  
45
	@Autowired
46
	private MongoDatabase objectstoreMongoDB;
47

  
48
	private String interpretation = "interp";
49
	private String basePath = "/tmp/basePath";
50
	//must be 36 chars
51
	private String obsID = "test56789111315171921232527293133350";
52
	private String pdfPath = "./test.pdf";
53

  
54
	@Before
55
	public void setup() throws IOException, ObjectStoreServiceException {
56
		Path baseDirPath = FileSystems.getDefault().getPath(basePath);
57
		if (!Files.exists(baseDirPath)) {
58
			Files.createDirectory(baseDirPath);
59
		}
60
		objectStoreDAO.createObjectStore(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl", interpretation, basePath);
61
	}
62

  
63
	@After
64
	public void tearDown() throws IOException, ObjectStoreServiceException {
65
		objectstoreMongoDB.drop();
66
		FileUtils.deleteDirectory(new File(basePath));
67
	}
68

  
69
	@Test
70
	public void testList() {
71
		final List<String> objectStores = this.objectStoreDAO.listObjectStores();
72
		for (String o : objectStores)
73
			System.out.println(o);
74
		assertTrue(objectStores.contains(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl"));
75
	}
76

  
77
	@Test
78
	public void testGet() throws ObjectStoreServiceException {
79
		ObjectStore o = objectStoreDAO.getObjectStore(obsID);
80
		System.out.println(o.toString());
81
		assertNotNull(o);
82
	}
83

  
84
	@Test
85
	public void testFeed() throws ObjectStoreServiceException, FileNotFoundException {
86
		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(),true);
87
	}
88

  
89
	@Test
90
	public void testDeliverObject() throws ObjectStoreServiceException {
91
		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
92
		final ObjectStoreFile obj2 = objectStoreDAO.getObjectStore(obsID).deliverObject("Oggetto_2");
93
		System.out.println(obj2.toJSON());
94
		assertNotNull(obj2);
95
	}
96

  
97
	@Test
98
	public void testGetStream() throws Exception {
99
		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
100
		ResultSetListener rs = objectStoreDAO.getObjectStore(obsID).deliver(0L, System.currentTimeMillis());
101
		for (int i=1; i< (rs.getSize()/10); ) {
102

  
103
			int from  = i;
104
			int to = Math.max(rs.getSize(), i*10);
105

  
106
			List<String> data = rs.getResult(from, to);
107

  
108
			for (String s: data) {
109
				System.out.println(s);
110
			}
111

  
112
			i= to ;
113
		}
114
	}
115

  
116

  
117
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java
1
/**
2
 *
3
 */
4
package eu.dnetlib.data.objectstore.filesystem;
5

  
6
import java.io.IOException;
7
import java.util.Iterator;
8

  
9
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
10
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
11
import eu.dnetlib.data.objectstore.rmi.Protocols;
12
import org.springframework.core.io.ClassPathResource;
13
import org.springframework.core.io.Resource;
14

  
15
/**
16
 * @author sandro
17
 *
18
 */
19
public class InputIterator implements Iterable<ObjectStoreRecord>, Iterator<ObjectStoreRecord> {
20

  
21
	private int counter = 0;
22
	private String pdfResourcePath = "test.pdf";
23
	private Resource testPDF = new ClassPathResource(pdfResourcePath);
24

  
25
	/**
26
	 * {@inheritDoc}
27
	 * @see java.util.Iterator#hasNext()
28
	 */
29
	@Override
30
	public boolean hasNext() {
31

  
32
		return counter<100;
33
	}
34

  
35
	/**
36
	 * {@inheritDoc}
37
	 * @see java.util.Iterator#next()
38
	 */
39
	@Override
40
	public ObjectStoreRecord next() {
41
		try {
42
			counter ++;
43
			ObjectStoreRecord record = new ObjectStoreRecord();
44
			ObjectStoreFile fileMetadata = new ObjectStoreFile();
45
			fileMetadata.setAccessProtocol(Protocols.File_System);
46

  
47
			fileMetadata.setDownloadedURL("file://" + pdfResourcePath);
48
			fileMetadata.setURI("file://" + pdfResourcePath);
49
			fileMetadata.setObjectID("Oggetto_"+counter);
50
			fileMetadata.setMimeType("application/pdf");
51
			System.out.println("Aggiungo Elemento "+counter);
52
			record.setInputStream(testPDF.getInputStream());
53
			record.setFileMetadata(fileMetadata );
54
			return record;
55
		} catch (IOException e) {
56
			throw new RuntimeException(e);
57
		}
58
	}
59

  
60
	/**
61
	 * {@inheritDoc}
62
	 * @see java.util.Iterator#remove()
63
	 */
64
	@Override
65
	public void remove() {
66
		// TODO Auto-generated method stub
67

  
68
	}
69

  
70
	/**
71
	 * {@inheritDoc}
72
	 * @see java.lang.Iterable#iterator()
73
	 */
74
	@Override
75
	public Iterator<ObjectStoreRecord> iterator() {
76
		counter =0;
77
		return this;
78
	}
79

  
80
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import java.net.UnknownHostException;
4

  
5
import com.mongodb.MongoClient;
6
import com.mongodb.client.MongoDatabase;
7
import org.springframework.context.annotation.Bean;
8
import org.springframework.context.annotation.Configuration;
9

  
10
// TODO: Auto-generated Javadoc
11
/**
12
 * The Class ConfigurationTestConfig.
13
 */
14
@Configuration
15
public class ConfigurationTestConfig {
16

  
17
	/**
18
	 * Mongo db.
19
	 *
20
	 * @return the db
21
	 * @throws UnknownHostException the unknown host exception
22
	 */
23
	@Bean
24
	public MongoDatabase objectstoreMongoDB() throws UnknownHostException {
25
		MongoClient mongoClient = new MongoClient("localhost");
26
		return mongoClient.getDatabase("objectStoreTest");
27

  
28
	}
29

  
30

  
31
	@Bean
32
	public FileSystemObjectStoreDao fsObjectStoreDAO() {
33
		FileSystemObjectStoreDao fileSystemObjectStoreDao = new FileSystemObjectStoreDao();
34

  
35
		fileSystemObjectStoreDao.setObjectStoreRESTURI("http://www.objectstore.com");
36
		return fileSystemObjectStoreDao;
37

  
38
	}
39

  
40
	/**
41
	 * Fs utility.
42
	 *
43
	 * @return the file system utility
44
	 */
45
	@Bean
46
	public FileSystemUtility fsUtility () {
47
		return new FileSystemUtility();
48
	}
49

  
50

  
51
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import java.io.*;
4
import java.net.URLEncoder;
5
import java.nio.file.Files;
6
import java.nio.file.Path;
7
import javax.servlet.http.HttpServletResponse;
8

  
9
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
10
import eu.dnetlib.miscutils.datetime.HumanTime;
11
import org.apache.commons.io.IOUtils;
12
import org.apache.commons.logging.Log;
13
import org.apache.commons.logging.LogFactory;
14
import org.springframework.stereotype.Controller;
15
import org.springframework.web.bind.annotation.RequestMapping;
16
import org.springframework.web.bind.annotation.RequestParam;
17

  
18
/**
19
 * The Class ModularObjectStoreRESTService implement the controller REST of the object Store.
20
 */
21
@Controller
22
public class ModularObjectStoreRESTService {
23

  
24
	private static final Log log = LogFactory.getLog(ModularObjectStoreRESTService.class); // NOPMD by marko on 11/24/08 5:02 PM
25

  
26
	public static String retrieveURL(final String baseURI, final String basePath, final String objectStoreId, final String objectId)
27
			throws UnsupportedEncodingException {
28
		final StringBuilder sb = new StringBuilder(baseURI)
29
				.append("?objectStore=" + encode(objectStoreId))
30
				.append("&objectId=" + encode(objectId))
31
				.append("&basePath=" + encode(basePath));
32
		return sb.toString();
33
	}
34

  
35
	private static String encode(final String s) throws UnsupportedEncodingException {
36
		return URLEncoder.encode(s, "UTF-8");
37
	}
38

  
39
	/**
40
	 *
41
	 * @param res
42
	 * @param basePath
43
	 * @param objectStoreId
44
	 * @param objectId
45
	 * @throws IOException
46
	 * @throws ObjectStoreServiceException
47
	 */
48
	@RequestMapping(value = "/**/objectStore/retrieve.do")
49
	public void retrieve(final HttpServletResponse res,
50
			@RequestParam(value = "basePath", required = true) final String basePath,
51
			@RequestParam(value = "objectStore", required = true) final String objectStoreId,
52
			@RequestParam(value = "objectId", required = true) final String objectId) throws IOException, ObjectStoreServiceException {
53

  
54
		final long start = System.currentTimeMillis();
55
		final Path path = FileSystemUtility.objectStoreFilePath(basePath, objectStoreId, objectId);
56

  
57
		if (!Files.exists(path) || !Files.isReadable(path)) {
58
			final String msg = String.format("Object with identifier: %s not found the in %s", objectId, path);
59
			res.sendError(HttpServletResponse.SC_NOT_FOUND, msg);
60
			log.warn(msg);
61
		} else {
62
			try (final InputStream is = new BufferedInputStream(new FileInputStream(path.toFile()))) {
63

  
64
				final long size = Files.size(path);
65

  
66
				res.setHeader("Content-Length", String.valueOf(size));
67
				IOUtils.copyLarge(is, res.getOutputStream());
68

  
69
				if (log.isDebugEnabled()) {
70
					log.debug(String.format("retrive.do completed in %s, objId: %s", HumanTime.exactly(System.currentTimeMillis() - start), objectId));
71
				}
72
			} catch (IOException e) {
73
				final String msg = "unable to close input Stream";
74
				res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg);
75
				log.error(msg, e);
76
			}
77
		}
78
	}
79
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import java.io.UnsupportedEncodingException;
4

  
5
import com.mongodb.DBObject;
6
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
7
import eu.dnetlib.data.objectstore.rmi.Protocols;
8
import eu.dnetlib.miscutils.functional.UnaryFunction;
9
import org.apache.commons.lang.StringUtils;
10
import org.apache.commons.logging.Log;
11
import org.apache.commons.logging.LogFactory;
12

  
13
/**
14
 * The Class ObjectStoreFileBuilder generates an objectStoreFile bean
15
 *
16
 */
17
public class ObjectStoreFileUtility {
18

  
19
	private static final int KB_SIZE = 1024;
20

  
21
	/** The Constant log. */
22
	private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);
23

  
24
	public static ObjectStoreFile build(final DBObject metadata, final String baseURI, final String objectStoreID, final String basePath) {
25

  
26
		String originalFile = (String) metadata.get("originalObject");
27
		ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);
28
		ObjectStoreFile newFile = new ObjectStoreFile();
29
		newFile.setObjectID((String) metadata.get("id"));
30
		newFile.setAccessProtocol(Protocols.HTTP);
31
		newFile.setMimeType((String) metadata.get("mime"));
32
		newFile.setMd5Sum((String) metadata.get("md5Sum"));
33
		try {
34
			newFile.setFileSizeKB(Long.parseLong(metadata.get("size").toString()) / KB_SIZE);
35
		} catch (Throwable e) {
36
			log.error("Error on getting file size", e);
37
		}
38
		if (originalFile != null) {
39
			newFile.setMetadataRelatedID(original.getMetadataRelatedID());
40
			if (StringUtils.isBlank(original.getDownloadedURL())) {
41
				newFile.setDownloadedURL(original.getURI());
42
			} else {
43
				newFile.setDownloadedURL(original.getDownloadedURL());
44
			}
45
		}
46
		try {
47
			newFile.setURI(ModularObjectStoreRESTService.retrieveURL(baseURI, basePath, objectStoreID, newFile.getObjectID()));
48
		} catch (UnsupportedEncodingException e) {
49
			log.error("Error on Build objectStoreFile ", e);
50
		}
51
		return newFile;
52
	}
53

  
54
	public static UnaryFunction<String, DBObject> asJSON(final String baseURI, final String objectStoreID, final String basePath) {
55
		return input -> build(input, baseURI, objectStoreID, basePath).toJSON();
56
	}
57
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStore.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import com.google.common.base.Function;
4
import com.google.common.collect.Iterables;
5
import com.google.common.collect.Lists;
6
import com.google.gson.Gson;
7
import com.mongodb.BasicDBObject;
8
import com.mongodb.DBObject;
9
import com.mongodb.client.MongoCollection;
10
import com.mongodb.client.model.Filters;
11
import com.mongodb.client.result.DeleteResult;
12
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
13
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
14
import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
15
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
16
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
17
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
18
import eu.dnetlib.enabling.resultset.ResultSetListener;
19
import eu.dnetlib.miscutils.collections.Pair;
20
import org.apache.commons.lang.StringUtils;
21
import org.apache.commons.logging.Log;
22
import org.apache.commons.logging.LogFactory;
23
import org.bson.conversions.Bson;
24

  
25
import java.io.ByteArrayInputStream;
26
import java.io.IOException;
27
import java.nio.file.FileSystems;
28
import java.nio.file.Files;
29
import java.nio.file.Path;
30
import java.util.regex.Pattern;
31

  
32
/**
33
 * The Class FileSystemObjectStore.
34
 *
35
 * @author sandro
36
 */
37
public class FileSystemObjectStore implements ObjectStore {
38

  
39
	/**
40
	 *
41
	 */
42
	private static final String URI_FIELD = "uri";
43

  
44
	/**
45
	 *
46
	 */
47
	private static final String FS_PATH_FIELD = "fsPath";
48

  
49
	/** The Constant log. */
50
	private static final Log log = LogFactory.getLog(FileSystemObjectStore.class); // NOPMD by marko on 11/24/08 5:02 PM
51

  
52
	/** The id. */
53
	private final String id;
54

  
55
	/** The interpretation. */
56
	private final String interpretation;
57

  
58
	/** The base path. */
59
	private final String basePath;
60

  
61
	/** The base uri. */
62
	private final String baseURI;
63

  
64
	/** The mongo metadata. */
65
	private final MongoCollection<DBObject> mongoMetadata;
66

  
67
	/**
68
	 * Instantiates a new file system object store.
69
	 *
70
	 * @param identifier
71
	 *            the identifier
72
	 * @param interpretation
73
	 *            the interpretation
74
	 * @param basePath
75
	 *            the base path
76
	 * @param mongoMetadata
77
	 *            the mongo metadata
78
	 * @param baseURI
79
	 *            the base uri
80
	 */
81
	public FileSystemObjectStore(final String identifier, final String interpretation, final String basePath, final MongoCollection<DBObject> mongoMetadata,
82
			final String baseURI) {
83
		this.id = identifier;
84
		this.basePath = basePath;
85
		this.interpretation = interpretation;
86
		this.mongoMetadata = mongoMetadata;
87
		this.baseURI = baseURI;
88
	}
89

  
90
	/**
91
	 * {@inheritDoc}
92
	 *
93
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId()
94
	 */
95
	@Override
96
	public String getId() {
97
		return this.id;
98
	}
99

  
100
	/**
101
	 * {@inheritDoc}
102
	 *
103
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation()
104
	 */
105
	@Override
106
	public String getInterpretation() {
107
		return this.interpretation;
108
	}
109

  
110
	/**
111
	 * {@inheritDoc}
112
	 *
113
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean)
114
	 */
115
	@Override
116
	public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) throws ObjectStoreServiceException {
117
		if (records == null)
118
			return 0;
119

  
120
		Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
121
		if (!Files.exists(baseDirPath))
122
			throw new ObjectStoreServiceException("Error can't feed objects because the folder " + baseDirPath + " does not exist");
123

  
124
		int addedCounter = 0;
125
		for (ObjectStoreRecord record : records) {
126
			String url = feedObject(record);
127
			if (StringUtils.isNotBlank(url)) {
128
				addedCounter++;
129
			}
130
		}
131
		return addedCounter;
132
	}
133

  
134
	/**
135
	 * {@inheritDoc}
136
	 *
137
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)
138
	 *
139
	 * This method handles the case of web crawl files and other cases when the metadata in mdstores are also the objects to put into the objectstores.
140
	 */
141
	@Override
142
	public int  feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException {
143
		Iterable<ObjectStoreRecord> it = Iterables.transform(records, new Function<MetadataObjectRecord, ObjectStoreRecord>() {
144
			@Override
145
			public ObjectStoreRecord apply(final MetadataObjectRecord metadataObjectRecord) {
146
				ObjectStoreRecord r = new ObjectStoreRecord();
147
				r.setInputStream(new ByteArrayInputStream(metadataObjectRecord.getRecord().getBytes()));
148
				ObjectStoreFile fileMetadata = new ObjectStoreFile();
149
				fileMetadata.setObjectID(metadataObjectRecord.getId());
150
				fileMetadata.setMimeType(metadataObjectRecord.getMime());
151
				r.setFileMetadata(fileMetadata);
152
				return r;
153
			}
154
		});
155
		return feed(it, incremental);
156
	}
157

  
158
	/**
159
	 * {@inheritDoc}
160
	 *
161
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)
162
	 */
163
	@Override
164
	public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {
165
		return feedObject(record);
166
	}
167

  
168
	private String feedObject(final ObjectStoreRecord record) {
169
		if (record != null) {
170
			String objectIdentifier = record.getFileMetadata().getObjectID();
171
			if (StringUtils.isNotBlank(objectIdentifier)) {
172
				final Path objResolvedPath = FileSystemUtility.objectStoreFilePath(basePath, id, objectIdentifier);
173

  
174
				if (Files.notExists(objResolvedPath)) {
175
					try {
176
						log.debug("Creation of folder " + objResolvedPath.getParent());
177
						Files.createDirectories(objResolvedPath.getParent());
178
						log.debug("Folder " + objResolvedPath.getParent() + " created");
179
						String md5Sum = null;
180
						Long size = new Long(0);
181
						if (record.getInputStream() != null) {
182
							Pair<String, Long> infos = FileSystemUtility.saveAndGenerateMD5(record.getInputStream(), objResolvedPath);
183
							md5Sum = infos.getKey();
184
							size = infos.getValue();
185
						}
186
						final String url =
187
								ModularObjectStoreRESTService.retrieveURL(getBaseURI(), getBasePath(), getId(), record.getFileMetadata().getObjectID());
188
						if (StringUtils.isNotBlank(md5Sum)) {
189
							double timestamp = System.currentTimeMillis();
190
							BasicDBObject metadata = new BasicDBObject();
191
							metadata.put("id", record.getFileMetadata().getObjectID());
192
							metadata.put("mime", record.getFileMetadata().getMimeType());
193
							metadata.put("originalObject", record.getFileMetadata().toJSON());
194
							metadata.put("timestamp", timestamp);
195
							metadata.put("md5Sum", md5Sum);
196
							metadata.put("size", size);
197
							metadata.put(FS_PATH_FIELD, objResolvedPath.toAbsolutePath().toString());
198
							metadata.put(URI_FIELD, url);
199
							log.debug("saving metadata object to the collection: " + metadata.toString());
200
							mongoMetadata.insertOne(metadata);
201
						}
202
						return url;
203
					} catch (Exception e) {
204
						log.error("Something bad happen on inserting Record", e);
205
						log.error("Record: " + new Gson().toJson(record.getFileMetadata()));
206
					} finally {
207
						if (record.getInputStream() != null) {
208
							try {
209
								record.getInputStream().close();
210
							} catch (Exception e) {
211
								log.error("Error on close inputStream", e);
212
							}
213
						}
214
					}
215
				} else {
216
					log.debug("The File in the path" + objResolvedPath.getParent() + "exists ");
217
				}
218
			}
219
		}
220
		log.warn("Record for object store is null");
221
		return null;
222
	}
223

  
224
	/**
225
	 * {@inheritDoc}
226
	 *
227
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Long, java.lang.Long)
228
	 */
229
	@Override
230
	public ResultSetListener deliver(final Long from, final Long until) throws ObjectStoreServiceException {
231
		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
232
		resultSet.setBaseURI(getBaseURI());
233
		resultSet.setMongoCollection(mongoMetadata);
234
		resultSet.setObjectStoreID(getId());
235
		resultSet.setFromDate(from);
236
		resultSet.setUntilDate(until);
237
		resultSet.setBasePath(getBasePath());
238
		return resultSet;
239
	}
240

  
241
	/**
242
	 * {@inheritDoc}
243
	 *
244
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable)
245
	 */
246
	@Override
247
	public ResultSetListener deliverIds(final Iterable<String> ids) throws ObjectStoreServiceException {
248
		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
249
		resultSet.setBaseURI(getBaseURI());
250
		resultSet.setMongoCollection(mongoMetadata);
251
		resultSet.setObjectStoreID(getId());
252
		resultSet.setRecords(Lists.newArrayList(ids));
253
		resultSet.setBasePath(basePath);
254
		return resultSet;
255
	}
256

  
257
	/**
258
	 * {@inheritDoc}
259
	 *
260
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String)
261
	 */
262
	@Override
263
	public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {
264
		Bson query = Filters.eq("id", objectId);
265
		DBObject resultQuery = mongoMetadata.find(query).first();
266
		checkAndGetFsPathField(resultQuery, objectId);
267
		return ObjectStoreFileUtility.build(resultQuery, getBaseURI(), getId(), basePath);
268
	}
269

  
270
	private String checkAndGetFsPathField(final DBObject resultQuery, final String objectId) throws ObjectStoreServiceException {
271
		if (resultQuery == null || !resultQuery.containsField(FS_PATH_FIELD))
272
			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found or missing " + FS_PATH_FIELD + " field");
273
		String pathStr = (String) resultQuery.get(FS_PATH_FIELD);
274
		if (StringUtils.isBlank(pathStr))
275
			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " with blank " + FS_PATH_FIELD);
276
		return pathStr;
277
	}
278

  
279
	/**
280
	 * {@inheritDoc}
281
	 *
282
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize()
283
	 */
284
	@Override
285
	public int getSize() throws ObjectStoreServiceException {
286
		return (int) mongoMetadata.count();
287
	}
288

  
289
	/**
290
	 * {@inheritDoc}
291
	 *
292
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String)
293
	 */
294
	@Override
295
	public void deleteObject(final String objectId) throws ObjectStoreServiceException {
296
		Bson query = Filters.eq("id", objectId);
297
		DBObject response = mongoMetadata.find(query).first();
298
		String pathStr = checkAndGetFsPathField(response, objectId);
299
		Path path = FileSystems.getDefault().getPath(pathStr);
300
		if (Files.notExists(path))
301
			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found in the assigned path " + path);
302
		try {
303
			Files.delete(path);
304
		} catch (IOException e) {
305
			throw new ObjectStoreServiceException("An error occurs on delete file ", e);
306
		}
307
		mongoMetadata.deleteOne(query);
308
	}
309

  
310
	/**
311
	 * {@inheritDoc}
312
	 *
313
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getObject(java.lang.String)
314
	 */
315
	@Override
316
	public String getObject(final String recordId) throws ObjectStoreServiceException {
317
		Bson query = Filters.eq("id", recordId);
318
		DBObject response = mongoMetadata.find(query).first();
319
		if (response == null || !response.containsField(URI_FIELD))
320
			return null;
321
		return (String) response.get(URI_FIELD);
322
	}
323

  
324
	/**
325
	 * {@inheritDoc}
326
	 *
327
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String)
328
	 */
329
	@Override
330
	public boolean existIDStartsWith(final String startId) throws ObjectStoreServiceException {
331
		Bson query = Filters.regex("id", Pattern.compile(startId));
332
		return mongoMetadata.count(query) > 0;
333
	}
334

  
335
	@Override
336
	public boolean dropContent() throws ObjectStoreServiceException {
337
		if (getBasePath() == null) {
338
			throw new ObjectStoreServiceException("Error on dropping object store base_path required");
339
		}
340
		final Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
341
		try {
342
			FileSystemUtility.deleteFolderRecursive(baseDirPath);
343
		} catch (IOException e) {
344
			throw new ObjectStoreServiceException("Error on dropping store ", e);
345
		}
346
		log.info("Deleted folder" + baseDirPath.toString());
347
		if (!Files.exists(baseDirPath)) {
348
			log.info("Recreating folder " + baseDirPath);
349
			try {
350
				Files.createDirectory(baseDirPath);
351
			} catch (IOException e) {
352
				throw new ObjectStoreServiceException("Error on dropping store ", e);
353
			}
354
		}
355
		final DeleteResult deleteResult = this.mongoMetadata.deleteMany(new BasicDBObject());
356
		log.info("Dropped content for object store " + id + ". " + deleteResult.getDeletedCount() + " object(s) deleted.");
357
		return true;
358
	}
359

  
360
	@Override
361
	public String toString() {
362
		return "FileSystemObjectStore{" +
363
				"id='" + getId() + '\'' +
364
				", interpretation='" + getInterpretation() + '\'' +
365
				", basePath='" + getBasePath() + '\'' +
366
				", baseURI='" + getBaseURI() + '\'' +
367
				'}';
368
	}
369

  
370
	/**
371
	 * Gets the base uri.
372
	 *
373
	 * @return the baseURI
374
	 */
375
	public String getBaseURI() {
376
		return baseURI;
377
	}
378

  
379
	public String getBasePath() {
380
		return basePath;
381
	}
382
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java
1
/**
2
 *
3
 */
4
package eu.dnetlib.data.objectstore.filesystem;
5

  
6
import eu.dnetlib.miscutils.collections.Pair;
7
import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
8
import org.apache.commons.lang.StringUtils;
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11

  
12
import java.io.FileInputStream;
13
import java.io.IOException;
14
import java.io.InputStream;
15
import java.nio.file.*;
16
import java.nio.file.attribute.BasicFileAttributes;
17

  
18
/**
19
 * The Class FileSystemUtility.
20
 *
21
 * @author sandro
22
 */
23
public class FileSystemUtility {
24

  
25
	private static final Log log = LogFactory.getLog(FileSystemUtility.class); // NOPMD by marko on 11/24/08 5:02 PM
26

  
27
	public static Pair<String, Long> saveAndGenerateMD5(final InputStream inputStream, final Path filePath) {
28
		if(inputStream==null) return null;
29

  
30
		String md5 = null;
31
		long size = 0;
32
		try {
33
			Files.copy(inputStream, filePath);
34

  
35
			FileInputStream fis = new FileInputStream(filePath.toFile());
36
			md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(fis);
37
			fis.close();
38
			size = Files.size(filePath);
39

  
40
		} catch (IOException e1) {
41
			log.error(e1);
42
		}
43

  
44
		return new Pair<String, Long>(md5, size);
45
	}
46

  
47
	/**
48
	 * Delete folder recursive.
49
	 *
50
	 * @param path the path
51
	 * @return true, if successful
52
	 * @throws IOException Signals that an I/O exception has occurred.
53
	 */
54
	public static boolean deleteFolderRecursive(final Path path) throws IOException {
55

  
56
		Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
57
			@Override
58
			public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
59
				Files.delete(file);
60
				return FileVisitResult.CONTINUE;
61
			}
62

  
63
			@Override
64
			public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
65
				Files.delete(dir);
66
				return FileVisitResult.CONTINUE;
67
			}
68
		});
69
		return true;
70
	}
71

  
72
	public static Path objectStoreFilePath(final String basePath, final String objectStoreId, final String objectIdentifier) {
73
		final Path baseDirPath = FileSystems.getDefault().getPath(basePath).resolve(objectStoreId);
74
		final String md5id = DnetXsltFunctions.md5(objectIdentifier);
75
		final String firstLevel = StringUtils.substring(md5id, 0, 2);
76
		final String secondLevel = StringUtils.substring(md5id, 2, 4);
77
		final String fileName = StringUtils.substring(md5id, 4) + ".obj";
78
		return baseDirPath.resolve(firstLevel).resolve(secondLevel).resolve(fileName);
79
	}
80

  
81
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreResultSetListener.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import java.util.List;
4

  
5
import com.google.common.collect.Lists;
6
import com.mongodb.DBObject;
7
import com.mongodb.client.FindIterable;
8
import com.mongodb.client.MongoCollection;
9
import com.mongodb.client.MongoCursor;
10
import com.mongodb.client.model.Filters;
11
import com.mongodb.client.model.Sorts;
12
import eu.dnetlib.enabling.resultset.ResultSet;
13
import eu.dnetlib.enabling.resultset.ResultSetAware;
14
import eu.dnetlib.enabling.resultset.ResultSetListener;
15
import eu.dnetlib.miscutils.collections.MappedCollection;
16
import org.apache.commons.logging.Log;
17
import org.apache.commons.logging.LogFactory;
18
import org.bson.conversions.Bson;
19

  
20
/**
21
 * The listener interface for receiving fileSystemObjectStoreResultSet events.
22
 * The class that is interested in processing a fileSystemObjectStoreResultSet
23
 * event implements this interface, and the object created
24
 * with that class is registered with a component using the
25
 * component's <code>addFileSystemObjectStoreResultSetListener<code> method. When
26
 * the fileSystemObjectStoreResultSet event occurs, that object's appropriate
27
 * method is invoked.
28
 *
29
 * @author sandro
30
 */
31
public class FileSystemObjectStoreResultSetListener implements ResultSetListener, ResultSetAware {
32

  
33

  
34
	/** The Constant log. */
35
	private static final Log log = LogFactory.getLog(FileSystemObjectStoreResultSetListener.class); // NOPMD by marko on 11/24/08 5:02 PM
36

  
37

  
38
	/** The from date. */
39
	private Long fromDate;
40

  
41
	/** The until date. */
42
	private Long untilDate;
43

  
44
	/** The records. */
45
	private List<String> records;
46

  
47
	/** The object store id. */
48
	private String objectStoreID;
49

  
50

  
51
	/** The mongo collection. */
52
	private MongoCollection<DBObject> mongoCollection;
53

  
54
	/** The base uri. */
55
	private String baseURI;
56

  
57
	/**
58
	 * The base path
59
	 */
60
	private String basePath;
61

  
62
	/** The current size. */
63
	private int currentSize = -1;
64

  
65
	/** The current cursor. */
66
	private MongoCursor<DBObject> currentCursor;
67

  
68
	/** The cursor position. */
69
	private long cursorPosition;
70

  
71
	/**
72
	 * {@inheritDoc}
73
	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)
74
	 */
75
	@Override
76
	public List<String> getResult(final int from, final int to) {
77
		if (log.isDebugEnabled()) {
78
			log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));
79
		}
80
		if (records != null) {
81
			List<String> ids = Lists.newArrayList();
82
			for (int i = from-1; i < Math.min(records.size(),to); i++) {
83
				ids.add(records.get(i));
84
			}
85
			Bson q = Filters.in("id", ids);
86
			FindIterable<DBObject> res = getMongoCollection().find(q);
87
			return MappedCollection.listMap(res, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
88
		} else if ((fromDate != null) && (untilDate != null)) {
89
			if ((currentCursor == null) || (cursorPosition > from)) {
90
				createCurrentCursor();
91
			}
92
			while (cursorPosition < from) {
93
				currentCursor.next();
94
				cursorPosition++;
95
			}
96
			List<DBObject> result = Lists.newArrayList();
97
			for (int i = from; i <= to; i++) {
98
				if (currentCursor.hasNext()) {
99
					result.add(currentCursor.next());
100
					cursorPosition++;
101
				}
102
			}
103
			return MappedCollection.listMap(result, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
104
		}
105

  
106
		throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");
107
	}
108

  
109
	/**
110
	 * Creates the current cursor.
111
	 */
112
	private void createCurrentCursor() {
113
		Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
114
		if (currentCursor != null) {
115
			currentCursor.close();
116
		}
117
		currentCursor = getMongoCollection().find(timestampQuery).sort(Sorts.orderBy(Filters.eq("_id", 1))).iterator();
118
		cursorPosition = 1;
119

  
120
	}
121

  
122
	/**
123
	 * {@inheritDoc}
124
	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()
125
	 */
126
	@Override
127
	public int getSize() {
128
		if (currentSize == -1) {
129
			currentSize = calculateSize();
130
		}
131
		return Math.max(0, currentSize - 1);
132
	}
133

  
134
	/**
135
	 * Calculate size.
136
	 *
137
	 * @return the int
138
	 */
139
	private int calculateSize() {
140
		if (records != null) {
141
			Bson query = Filters.in("id", records);
142
			return (int) getMongoCollection().count(query);
143
		} else if ((fromDate != null) && (untilDate != null)) {
144
			Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
145
			return (int) getMongoCollection().count(timestampQuery);
146
		}
147
		return 0;
148
	}
149

  
150

  
151
	/**
152
	 * {@inheritDoc}
153
	 * @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)
154
	 */
155
	@Override
156
	public void setResultSet(final ResultSet resultSet) {
157
		resultSet.close();
158
	}
159

  
160

  
161
	/**
162
	 * Gets the from date.
163
	 *
164
	 * @return the fromDate
165
	 */
166
	public Long getFromDate() {
167
		return fromDate;
168
	}
169

  
170

  
171
	/**
172
	 * Sets the from date.
173
	 *
174
	 * @param fromDate the fromDate to set
175
	 */
176
	public FileSystemObjectStoreResultSetListener setFromDate(final Long fromDate) {
177
		this.fromDate = fromDate;
178
		return this;
179
	}
180

  
181

  
182
	/**
183
	 * Gets the until date.
184
	 *
185
	 * @return the untilDate
186
	 */
187
	public Long getUntilDate() {
188
		return untilDate;
189
	}
190

  
191

  
192
	/**
193
	 * Sets the until date.
194
	 *
195
	 * @param untilDate the untilDate to set
196
	 */
197
	public FileSystemObjectStoreResultSetListener setUntilDate(final Long untilDate) {
198
		this.untilDate = untilDate;
199
		return this;
200
	}
201

  
202

  
203
	/**
204
	 * Gets the records.
205
	 *
206
	 * @return the records
207
	 */
208
	public List<String> getRecords() {
209
		return records;
210
	}
211

  
212

  
213
	/**
214
	 * Sets the records.
215
	 *
216
	 * @param records the records to set
217
	 */
218
	public void setRecords(final List<String> records) {
219
		this.records = records;
220
	}
221

  
222

  
223
	/**
224
	 * Gets the object store id.
225
	 *
226
	 * @return the objectStoreID
227
	 */
228
	public String getObjectStoreID() {
229
		return objectStoreID;
230
	}
231

  
232

  
233
	/**
234
	 * Sets the object store id.
235
	 *
236
	 * @param objectStoreID the objectStoreID to set
237
	 */
238
	public void setObjectStoreID(final String objectStoreID) {
239
		this.objectStoreID = objectStoreID;
240
	}
241

  
242

  
243

  
244

  
245

  
246
	/**
247
	 * Gets the base uri.
248
	 *
249
	 * @return the baseURI
250
	 */
251
	public String getBaseURI() {
252
		return baseURI;
253
	}
254

  
255

  
256
	/**
257
	 * Sets the base uri.
258
	 *
259
	 * @param baseURI the baseURI to set
260
	 */
261
	public void setBaseURI(final String baseURI) {
262
		this.baseURI = baseURI;
263
	}
264

  
265

  
266
	/**
267
	 * Gets the current size.
268
	 *
269
	 * @return the currentSize
270
	 */
271
	public int getCurrentSize() {
272
		return currentSize;
273
	}
274

  
275

  
276
	/**
277
	 * Sets the current size.
278
	 *
279
	 * @param currentSize the currentSize to set
280
	 */
281
	public void setCurrentSize(final int currentSize) {
282
		this.currentSize = currentSize;
283
	}
284

  
285

  
286
	/**
287
	 * Gets the current cursor.
288
	 *
289
	 * @return the currentCursor
290
	 */
291
	public MongoCursor<DBObject> getCurrentCursor() {
292
		return currentCursor;
293
	}
294

  
295

  
296
	/**
297
	 * Sets the current cursor.
298
	 *
299
	 * @param currentCursor the currentCursor to set
300
	 */
301
	public void setCurrentCursor(final MongoCursor<DBObject> currentCursor) {
302
		this.currentCursor = currentCursor;
303
	}
304

  
305

  
306
	/**
307
	 * Gets the cursor position.
308
	 *
309
	 * @return the cursorPosition
310
	 */
311
	public long getCursorPosition() {
312
		return cursorPosition;
313
	}
314

  
315

  
316
	/**
317
	 * Sets the cursor position.
318
	 *
319
	 * @param cursorPosition the cursorPosition to set
320
	 */
321
	public void setCursorPosition(final long cursorPosition) {
322
		this.cursorPosition = cursorPosition;
323
	}
324

  
325
	/**
326
	 * Gets the mongo collection.
327
	 *
328
	 * @return the mongo collection
329
	 */
330
	public MongoCollection<DBObject> getMongoCollection() {
331
		return mongoCollection;
332
	}
333

  
334
	/**
335
	 * Sets the mongo collection.
336
	 *
337
	 * @param mongoCollection the new mongo collection
338
	 */
339
	public void setMongoCollection(final MongoCollection<DBObject> mongoCollection) {
340
		this.mongoCollection = mongoCollection;
341
	}
342

  
343
	public String getBasePath() {
344
		return basePath;
345
	}
346

  
347
	public void setBasePath(final String basePath) {
348
		this.basePath = basePath;
349
	}
350
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java
1
package eu.dnetlib.data.objectstore.filesystem;
2

  
3
import com.mongodb.BasicDBObject;
4
import com.mongodb.DBObject;
5
import com.mongodb.client.MongoCollection;
6
import com.mongodb.client.MongoDatabase;
7
import com.mongodb.client.model.IndexOptions;
8
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
9
import org.apache.commons.lang.StringUtils;
10
import org.apache.commons.logging.Log;
11
import org.apache.commons.logging.LogFactory;
12
import org.springframework.beans.factory.annotation.Autowired;
13

  
14
import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_ID_FIELD;
15
import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_METADATA_NAME_FIELD;
16

  
17
/**
18
 * Created by claudio on 15/09/16.
19
 */
20
public class IndexIntegrityChecker {
21

  
22
	private static final Log log = LogFactory.getLog(IndexIntegrityChecker.class);
23

  
24
	@Autowired
25
	private ObjectStoreDao dao;
26

  
27
	public void check() {
28
		checkIndexes();
29
	}
30

  
31
	private void checkIndexes() {
32
		log.info("objectStore indexes integrity start");
33

  
34
		final MongoDatabase db = ((FileSystemObjectStoreDao) dao).getDb();
35
		final IndexOptions bg = new IndexOptions().background(true);
36

  
37
		for (String objectStoreId : dao.listObjectStores()) {
38
			final String id = StringUtils.substringBefore(objectStoreId, "_");
39
			final MongoCollection<DBObject> objectStore = db.getCollection(id, DBObject.class);
40
			if (log.isDebugEnabled()) {
41
				log.debug(String.format("creating index (id, timestamp) on objectStore %s", id));
42
			}
43
			objectStore.createIndex(new BasicDBObject("id", 1), bg);
44
			objectStore.createIndex(new BasicDBObject("timestamp", 1), bg);
45
		}
46
		if (log.isDebugEnabled()) {
47
			log.debug(String.format("creating index (%s) on %s", OBJECTSTORE_ID_FIELD, OBJECTSTORE_METADATA_NAME_FIELD));
48
		}
49
		final MongoCollection<DBObject> metadata = db.getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
50
		metadata.createIndex(new BasicDBObject(OBJECTSTORE_ID_FIELD, 1), bg);
51

  
52
		log.info("objectStore indexes integrity completed");
53
	}
54

  
55
}
modules/dnet-fs-objectstore/tags/dnet-fs-objectstore-2.1.8/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreDao.java
1
/**
2
 *
3
 */
4
package eu.dnetlib.data.objectstore.filesystem;
5

  
6
import java.io.IOException;
7
import java.nio.file.FileSystems;
8
import java.nio.file.Files;
9
import java.nio.file.Path;
10
import java.util.List;
11
import javax.annotation.Resource;
12

  
13
import com.mongodb.BasicDBObject;
14
import com.mongodb.DBObject;
15
import com.mongodb.client.MongoCollection;
16
import com.mongodb.client.MongoDatabase;
17
import com.mongodb.client.model.Filters;
18
import com.mongodb.client.result.UpdateResult;
19
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
20
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
21
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
22
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
23
import eu.dnetlib.miscutils.collections.MappedCollection;
24
import eu.dnetlib.miscutils.functional.UnaryFunction;
25
import org.apache.commons.lang.StringUtils;
26
import org.apache.commons.logging.Log;
27
import org.apache.commons.logging.LogFactory;
28
import org.bson.conversions.Bson;
29

  
30
/**
31
 * @author sandro
32
 *
33
 */
34
public class FileSystemObjectStoreDao implements ObjectStoreDao {
35

  
36
	public static final String INTERPRETATION_FIELD = "interpretation";
37
	public final static String OBJECTSTORE_METADATA_NAME_FIELD = "metadataObjectStore";
38
	public final static String OBJECTSTORE_ID_FIELD = "obsId";
39
	public final static String BASE_PATH_FIELD = "basePath";
40
	private static final Log log = LogFactory.getLog(FileSystemObjectStoreDao.class); // NOPMD by marko on 11/24/08 5:02 PM
41
	private static final String OBJECTSTORE_PROFILE_SUFFIX = "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl";
42
	@Resource(name="objectstoreMongoDB")
43
	private MongoDatabase db;
44

  
45
	private boolean upsert;
46

  
47
	private String objectStoreRESTURI;
48

  
49
	/**
50
	 * {@inheritDoc}
51
	 * @throws ObjectStoreServiceException
52
	 *
53
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#getObjectStore(java.lang.String)
54
	 */
55
	@Override
56
	public ObjectStore getObjectStore(final String obsId) throws ObjectStoreServiceException {
57
		String currentId = obsId.substring(0, 36);
58
		String find_id = obsId;
59
		if (find_id.length() == 36) {
60
			find_id += OBJECTSTORE_PROFILE_SUFFIX;
61
		}
62

  
63
		MongoCollection<DBObject> metadataObjectStore = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
64
		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, find_id);
65
		log.debug("QUERY :" + query.toString());
66
		DBObject resultQuery = metadataObjectStore.find(query).first();
67
		log.debug("result " + resultQuery);
68
		if ((resultQuery == null)) throw new ObjectStoreFileNotFoundException("the objectStore with identifier: "+obsId+" was not found");
69

  
70
		final String basePath = resultQuery.get(BASE_PATH_FIELD).toString();
71
		final String interpretation = resultQuery.get("interpretation").toString();
72

  
73
		if (!resultQuery.containsField(BASE_PATH_FIELD) || StringUtils.isBlank(basePath))
74
			throw new ObjectStoreServiceException("Can't Get Objectstore, the metadata doesn't contain info about the basepath");
75

  
76
		final MongoCollection<DBObject> collection = getDb().getCollection(currentId, DBObject.class);
77
		return new FileSystemObjectStore(currentId, interpretation, basePath, collection, objectStoreRESTURI);
78
	}
79

  
80
	/**
81
	 * {@inheritDoc}
82
	 *
83
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#listObjectStores()
84
	 */
85
	@Override
86
	public List<String> listObjectStores() {
87
		MongoCollection<DBObject> metadata = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
88
		return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() {
89

  
90
			@Override
91
			public String evaluate(final DBObject object) {
92
				return (String) object.get(OBJECTSTORE_ID_FIELD);
93
			}
94
		});
95
	}
96

  
97
	/**
98
	 * {@inheritDoc}
99
	 *
100
	 * @throws ObjectStoreServiceException
101
	 */
102
	@Override
103
	public boolean createObjectStore(final String obsId, final String interpretation, final String basePath) throws ObjectStoreServiceException {
104

  
105
		log.debug(String.format("Create object Store method\n\t Id:%s  Interpretation:%s BasePath : %s", obsId, interpretation, basePath) );
106

  
107
		if (StringUtils.isBlank(basePath)) throw new ObjectStoreServiceException("Can't create the object store: the base path cannot be blank");
108
		Path path = FileSystems.getDefault().getPath(basePath);
109
		if (!Files.exists(path) || !Files.isDirectory(path))
110
			throw new ObjectStoreServiceException("Can't create the object store: base path: '" + basePath + "' doesn't exist or it is not a folder");
111
		try {
112
			String currentObsId = obsId.substring(0, 36);
113
			log.debug("Cleaned objectStore Id " + currentObsId);
114
			if (Files.exists(path.resolve(currentObsId)))
115
				throw new ObjectStoreServiceException("Can't create the object store: base path: '" + path.resolve(currentObsId) + "' already exists");
116
			Files.createDirectory(path.resolve(currentObsId));
117
			MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
118
			final BasicDBObject obj = new BasicDBObject();
119
			obj.put(OBJECTSTORE_ID_FIELD, obsId);
120
			obj.put(INTERPRETATION_FIELD, interpretation);
121
			obj.put(BASE_PATH_FIELD, basePath);
122
			coll.insertOne(obj);
123
			MongoCollection<DBObject> objectStore = getDb().getCollection(currentObsId, DBObject.class);
124
			objectStore.createIndex(new BasicDBObject("id", 1));
125
			objectStore.createIndex(new BasicDBObject("timestamp", 1));
126
			return true;
127
		} catch (Throwable e) {
128
			throw new ObjectStoreServiceException("Can't Create the object Store id: '" + obsId, e);
129
		}
130
	}
131

  
132
	/**
133
	 * {@inheritDoc}
134
	 *
135
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#updateObjectStore(java.lang.String, java.lang.String)
136
	 */
137
	@Override
138
	public boolean updateObjectStore(final String obsId, final String interpretation) {
139
		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
140
		final BasicDBObject obj = new BasicDBObject();
141
		obj.put("$set", new BasicDBObject(INTERPRETATION_FIELD, interpretation));
142

  
143
		final UpdateResult updateResult = coll.updateOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId), obj);
144
		if (updateResult.isModifiedCountAvailable()) {
145
			log.debug("Matched / Modified " + updateResult.getMatchedCount() + " / " + updateResult.getModifiedCount());
146
		}
147
		return true;
148
	}
149

  
150
	/**
151
	 * {@inheritDoc}
152
	 *
153
	 * @throws ObjectStoreServiceException
154
	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#deleteObjectStore(java.lang.String)
155
	 */
156
	@Override
157
	public boolean deleteObjectStore(final String obsId) throws ObjectStoreServiceException {
158
		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
159
		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, obsId);
160
		DBObject resultQuery = coll.find(query).first();
161
		String basePath = checkAndGetFsPathField(resultQuery, obsId);
162
		String currentObsId = obsId.substring(0, 36);
163
		Path basePathFS = FileSystems.getDefault().getPath(basePath, currentObsId);
164
		if (!Files.exists(basePathFS))
165
			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId + ": the base path does not exist :" + basePathFS);
166
		try {
167
			FileSystemUtility.deleteFolderRecursive(basePathFS);
168
		} catch (IOException e) {
169
			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId, e);
170
		}
171
		coll.deleteOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId));
172
		getDb().getCollection(obsId).drop();
173
		return true;
174
	}
175

  
176
	@Override
177
	public boolean dropContent(final String obsId) throws ObjectStoreServiceException {
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff