D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-fs-objectstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-fs-objectstore"}

package eu.dnetlib.data.objectstore.filesystem;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.IOException;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.List;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import org.apache.commons.io.FileUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.After;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

/**

 * TODO: Test class set to ignored because it requires a mongo server running on localhost. Tests with mocks should be prepared.

 * @author sandro

 *

 */

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class ObjectStoreServiceDAOTest {

	private static final Log log = LogFactory.getLog(ObjectStoreServiceDAOTest.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Autowired

	private FileSystemObjectStoreDao objectStoreDAO;

	@Autowired

	private MongoDatabase objectstoreMongoDB;

	private String interpretation = "interp";

	private String basePath = "/tmp/basePath";

	//must be 36 chars

	private String obsID = "test56789111315171921232527293133350";

	private String pdfPath = "./test.pdf";

	@Before

	public void setup() throws IOException, ObjectStoreServiceException {

		Path baseDirPath = FileSystems.getDefault().getPath(basePath);

		if (!Files.exists(baseDirPath)) {

			Files.createDirectory(baseDirPath);

		}

		objectStoreDAO.createObjectStore(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl", interpretation, basePath);

	}

	@After

	public void tearDown() throws IOException, ObjectStoreServiceException {

		objectstoreMongoDB.drop();

		FileUtils.deleteDirectory(new File(basePath));

	}

	@Test

	public void testList() {

		final List<String> objectStores = this.objectStoreDAO.listObjectStores();

		for (String o : objectStores)

			System.out.println(o);

		assertTrue(objectStores.contains(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl"));

	}

	@Test

	public void testGet() throws ObjectStoreServiceException {

		ObjectStore o = objectStoreDAO.getObjectStore(obsID);

		System.out.println(o.toString());

		assertNotNull(o);

	}

	@Test

	public void testFeed() throws ObjectStoreServiceException, FileNotFoundException {

		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(),true);

	}

	@Test

	public void testDeliverObject() throws ObjectStoreServiceException {

		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);

		final ObjectStoreFile obj2 = objectStoreDAO.getObjectStore(obsID).deliverObject("Oggetto_2");

		System.out.println(obj2.toJSON());

		assertNotNull(obj2);

	}

	@Test

	public void testGetStream() throws Exception {

		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);

		ResultSetListener rs = objectStoreDAO.getObjectStore(obsID).deliver(0L, System.currentTimeMillis());

		for (int i=1; i< (rs.getSize()/10); ) {

			int from  = i;

			int to = Math.max(rs.getSize(), i*10);

			List<String> data = rs.getResult(from, to);

			for (String s: data) {

				System.out.println(s);

			}

			i= to ;

		}

	}

}

/**

 *

 */

package eu.dnetlib.data.objectstore.filesystem;

import java.io.IOException;

import java.util.Iterator;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

/**

 * @author sandro

 *

 */

public class InputIterator implements Iterable<ObjectStoreRecord>, Iterator<ObjectStoreRecord> {

	private int counter = 0;

	private String pdfResourcePath = "test.pdf";

	private Resource testPDF = new ClassPathResource(pdfResourcePath);

	/**

	 * {@inheritDoc}

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

		return counter<100;

	}

	/**

	 * {@inheritDoc}

	 * @see java.util.Iterator#next()

	 */

	@Override

	public ObjectStoreRecord next() {

		try {

			counter ++;

			ObjectStoreRecord record = new ObjectStoreRecord();

			ObjectStoreFile fileMetadata = new ObjectStoreFile();

			fileMetadata.setAccessProtocol(Protocols.File_System);

			fileMetadata.setDownloadedURL("file://" + pdfResourcePath);

			fileMetadata.setURI("file://" + pdfResourcePath);

			fileMetadata.setObjectID("Oggetto_"+counter);

			fileMetadata.setMimeType("application/pdf");

			System.out.println("Aggiungo Elemento "+counter);

			record.setInputStream(testPDF.getInputStream());

			record.setFileMetadata(fileMetadata );

			return record;

		} catch (IOException e) {

			throw new RuntimeException(e);

		}

	}

	/**

	 * {@inheritDoc}

	 * @see java.util.Iterator#remove()

	 */

	@Override

	public void remove() {

		// TODO Auto-generated method stub

	}

	/**

	 * {@inheritDoc}

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<ObjectStoreRecord> iterator() {

		counter =0;

		return this;

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import java.net.UnknownHostException;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoDatabase;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

// TODO: Auto-generated Javadoc

/**

 * The Class ConfigurationTestConfig.

 */

@Configuration

public class ConfigurationTestConfig {

	/**

	 * Mongo db.

	 *

	 * @return the db

	 * @throws UnknownHostException the unknown host exception

	 */

	@Bean

	public MongoDatabase objectstoreMongoDB() throws UnknownHostException {

		MongoClient mongoClient = new MongoClient("localhost");

		return mongoClient.getDatabase("objectStoreTest");

	}

	@Bean

	public FileSystemObjectStoreDao fsObjectStoreDAO() {

		FileSystemObjectStoreDao fileSystemObjectStoreDao = new FileSystemObjectStoreDao();

		fileSystemObjectStoreDao.setObjectStoreRESTURI("http://www.objectstore.com");

		return fileSystemObjectStoreDao;

	}

	/**

	 * Fs utility.

	 *

	 * @return the file system utility

	 */

	@Bean

	public FileSystemUtility fsUtility () {

		return new FileSystemUtility();

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import java.io.*;

import java.net.URLEncoder;

import java.nio.file.Files;

import java.nio.file.Path;

import javax.servlet.http.HttpServletResponse;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.miscutils.datetime.HumanTime;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestParam;

/**

 * The Class ModularObjectStoreRESTService implement the controller REST of the object Store.

 */

@Controller

public class ModularObjectStoreRESTService {

	private static final Log log = LogFactory.getLog(ModularObjectStoreRESTService.class); // NOPMD by marko on 11/24/08 5:02 PM

	public static String retrieveURL(final String baseURI, final String basePath, final String objectStoreId, final String objectId)

			throws UnsupportedEncodingException {

		final StringBuilder sb = new StringBuilder(baseURI)

				.append("?objectStore=" + encode(objectStoreId))

				.append("&objectId=" + encode(objectId))

				.append("&basePath=" + encode(basePath));

		return sb.toString();

	}

	private static String encode(final String s) throws UnsupportedEncodingException {

		return URLEncoder.encode(s, "UTF-8");

	}

	/**

	 *

	 * @param res

	 * @param basePath

	 * @param objectStoreId

	 * @param objectId

	 * @throws IOException

	 * @throws ObjectStoreServiceException

	 */

	@RequestMapping(value = "/**/objectStore/retrieve.do")

	public void retrieve(final HttpServletResponse res,

			@RequestParam(value = "basePath", required = true) final String basePath,

			@RequestParam(value = "objectStore", required = true) final String objectStoreId,

			@RequestParam(value = "objectId", required = true) final String objectId) throws IOException, ObjectStoreServiceException {

		final long start = System.currentTimeMillis();

		final Path path = FileSystemUtility.objectStoreFilePath(basePath, objectStoreId, objectId);

		if (!Files.exists(path) || !Files.isReadable(path)) {

			final String msg = String.format("Object with identifier: %s not found the in %s", objectId, path);

			res.sendError(HttpServletResponse.SC_NOT_FOUND, msg);

			log.warn(msg);

		} else {

			try (final InputStream is = new BufferedInputStream(new FileInputStream(path.toFile()))) {

				final long size = Files.size(path);

				res.setHeader("Content-Length", String.valueOf(size));

				IOUtils.copyLarge(is, res.getOutputStream());

				if (log.isDebugEnabled()) {

					log.debug(String.format("retrive.do completed in %s, objId: %s", HumanTime.exactly(System.currentTimeMillis() - start), objectId));

				}

			} catch (IOException e) {

				final String msg = "unable to close input Stream";

				res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg);

				log.error(msg, e);

			}

		}

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import java.io.UnsupportedEncodingException;

import com.mongodb.DBObject;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * The Class ObjectStoreFileBuilder generates an objectStoreFile bean

 *

 */

public class ObjectStoreFileUtility {

	private static final int KB_SIZE = 1024;

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);

	public static ObjectStoreFile build(final DBObject metadata, final String baseURI, final String objectStoreID, final String basePath) {

		String originalFile = (String) metadata.get("originalObject");

		ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);

		ObjectStoreFile newFile = new ObjectStoreFile();

		newFile.setObjectID((String) metadata.get("id"));

		newFile.setAccessProtocol(Protocols.HTTP);

		newFile.setMimeType((String) metadata.get("mime"));

		newFile.setMd5Sum((String) metadata.get("md5Sum"));

		try{

			newFile.setFileSizeKB(Long.parseLong(metadata.get("size").toString()) / KB_SIZE);

		} catch (Throwable e) {

			log.error("Error on getting size",e);

		}

		if (originalFile != null) {

			newFile.setMetadataRelatedID(original.getMetadataRelatedID());

			if ((original.getDownloadedURL() == null) || (original.getDownloadedURL().length() == 0)) {

				newFile.setDownloadedURL(original.getURI());

			} else {

				newFile.setDownloadedURL(original.getDownloadedURL());

			}

		}

		try {

			newFile.setURI(ModularObjectStoreRESTService.retrieveURL(baseURI, basePath, objectStoreID, newFile.getObjectID()));

		} catch (UnsupportedEncodingException e) {

			log.error("Error on Build objectStoreFile ", e);

		}

		return newFile;

	}

	public static UnaryFunction<String, DBObject> asJSON(final String baseURI, final String objectStoreID, final String basePath) {

		return new UnaryFunction<String, DBObject>() {

			@Override

			public String evaluate(final DBObject input) {

				return build(input, baseURI, objectStoreID, basePath).toJSON();

			}

		};

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.gson.Gson;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.model.Filters;

import com.mongodb.client.result.DeleteResult;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.Pair;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.conversions.Bson;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.regex.Pattern;

/**

 * The Class FileSystemObjectStore.

 *

 * @author sandro

 */

public class FileSystemObjectStore implements ObjectStore {

	/**

	 *

	 */

	private static final String URI_FIELD = "uri";

	/**

	 *

	 */

	private static final String FS_PATH_FIELD = "fsPath";

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(FileSystemObjectStore.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The id. */

	private final String id;

	/** The interpretation. */

	private final String interpretation;

	/** The base path. */

	private final String basePath;

	/** The base uri. */

	private final String baseURI;

	/** The mongo metadata. */

	private final MongoCollection<DBObject> mongoMetadata;

	/**

	 * Instantiates a new file system object store.

	 *

	 * @param identifier

	 *            the identifier

	 * @param interpretation

	 *            the interpretation

	 * @param basePath

	 *            the base path

	 * @param mongoMetadata

	 *            the mongo metadata

	 * @param baseURI

	 *            the base uri

	 */

	public FileSystemObjectStore(final String identifier, final String interpretation, final String basePath, final MongoCollection<DBObject> mongoMetadata,

			final String baseURI) {

		this.id = identifier;

		this.basePath = basePath;

		this.interpretation = interpretation;

		this.mongoMetadata = mongoMetadata;

		this.baseURI = baseURI;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId()

	 */

	@Override

	public String getId() {

		return this.id;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation()

	 */

	@Override

	public String getInterpretation() {

		return this.interpretation;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean)

	 */

	@Override

	public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) throws ObjectStoreServiceException {

		if (records == null)

			return 0;

		Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());

		if (!Files.exists(baseDirPath))

			throw new ObjectStoreServiceException("Error can't feed objects because the folder " + baseDirPath + " does not exist");

		int addedCounter = 0;

		for (ObjectStoreRecord record : records) {

			String url = feedObject(record);

			if (StringUtils.isNotBlank(url)) {

				addedCounter++;

			}

		}

		return addedCounter;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)

	 *

	 * This method handles the case of web crawl files and other cases when the metadata in mdstores are also the objects to put into the objectstores.

	 */

	@Override

	public int  feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException {

		Iterable<ObjectStoreRecord> it = Iterables.transform(records, new Function<MetadataObjectRecord, ObjectStoreRecord>() {

			@Override

			public ObjectStoreRecord apply(final MetadataObjectRecord metadataObjectRecord) {

				ObjectStoreRecord r = new ObjectStoreRecord();

				r.setInputStream(new ByteArrayInputStream(metadataObjectRecord.getRecord().getBytes()));

				ObjectStoreFile fileMetadata = new ObjectStoreFile();

				fileMetadata.setObjectID(metadataObjectRecord.getId());

				fileMetadata.setMimeType(metadataObjectRecord.getMime());

				r.setFileMetadata(fileMetadata);

				return r;

			}

		});

		return feed(it, incremental);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)

	 */

	@Override

	public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {

		return feedObject(record);

	}

	private String feedObject(final ObjectStoreRecord record) {

		if (record != null) {

			String objectIdentifier = record.getFileMetadata().getObjectID();

			if (StringUtils.isNotBlank(objectIdentifier)) {

				final Path objResolvedPath = FileSystemUtility.objectStoreFilePath(basePath, id, objectIdentifier);

				if (Files.notExists(objResolvedPath)) {

					try {

						log.debug("Creation of folder " + objResolvedPath.getParent());

						Files.createDirectories(objResolvedPath.getParent());

						log.debug("Folder " + objResolvedPath.getParent() + " created");

						String md5Sum = null;

						Long size = new Long(0);

						if (record.getInputStream() != null) {

							Pair<String, Long> infos = FileSystemUtility.saveAndGenerateMD5(record.getInputStream(), objResolvedPath);

							md5Sum = infos.getKey();

							size = infos.getValue();

						}

						final String url =

								ModularObjectStoreRESTService.retrieveURL(getBaseURI(), getBasePath(), getId(), record.getFileMetadata().getObjectID());

						if (StringUtils.isNotBlank(md5Sum)) {

							double timestamp = System.currentTimeMillis();

							BasicDBObject metadata = new BasicDBObject();

							metadata.put("id", record.getFileMetadata().getObjectID());

							metadata.put("mime", record.getFileMetadata().getMimeType());

							metadata.put("originalObject", record.getFileMetadata().toJSON());

							metadata.put("timestamp", timestamp);

							metadata.put("md5Sum", md5Sum);

							metadata.put("size", size);

							metadata.put(FS_PATH_FIELD, objResolvedPath.toAbsolutePath().toString());

							metadata.put(URI_FIELD, url);

							log.debug("saving metadata object to the collection: " + metadata.toString());

							mongoMetadata.insertOne(metadata);

						}

						return url;

					} catch (Exception e) {

						log.error("Something bad happen on inserting Record", e);

						log.error("Record: " + new Gson().toJson(record.getFileMetadata()));

					} finally {

						if (record.getInputStream() != null) {

							try {

								record.getInputStream().close();

							} catch (Exception e) {

								log.error("Error on close inputStream", e);

							}

						}

					}

				} else {

					log.debug("The File in the path" + objResolvedPath.getParent() + "exists ");

				}

			}

		}

		log.warn("Record for object store is null");

		return null;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Long, java.lang.Long)

	 */

	@Override

	public ResultSetListener deliver(final Long from, final Long until) throws ObjectStoreServiceException {

		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();

		resultSet.setBaseURI(getBaseURI());

		resultSet.setMongoCollection(mongoMetadata);

		resultSet.setObjectStoreID(getId());

		resultSet.setFromDate(from);

		resultSet.setUntilDate(until);

		resultSet.setBasePath(getBasePath());

		return resultSet;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable)

	 */

	@Override

	public ResultSetListener deliverIds(final Iterable<String> ids) throws ObjectStoreServiceException {

		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();

		resultSet.setBaseURI(getBaseURI());

		resultSet.setMongoCollection(mongoMetadata);

		resultSet.setObjectStoreID(getId());

		resultSet.setRecords(Lists.newArrayList(ids));

		resultSet.setBasePath(basePath);

		return resultSet;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String)

	 */

	@Override

	public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {

		Bson query = Filters.eq("id", objectId);

		DBObject resultQuery = mongoMetadata.find(query).first();

		checkAndGetFsPathField(resultQuery, objectId);

		return ObjectStoreFileUtility.build(resultQuery, getBaseURI(), getId(), basePath);

	}

	private String checkAndGetFsPathField(final DBObject resultQuery, final String objectId) throws ObjectStoreServiceException {

		if (resultQuery == null || !resultQuery.containsField(FS_PATH_FIELD))

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found or missing " + FS_PATH_FIELD + " field");

		String pathStr = (String) resultQuery.get(FS_PATH_FIELD);

		if (StringUtils.isBlank(pathStr))

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " with blank " + FS_PATH_FIELD);

		return pathStr;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize()

	 */

	@Override

	public int getSize() throws ObjectStoreServiceException {

		return (int) mongoMetadata.count();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String)

	 */

	@Override

	public void deleteObject(final String objectId) throws ObjectStoreServiceException {

		Bson query = Filters.eq("id", objectId);

		DBObject response = mongoMetadata.find(query).first();

		String pathStr = checkAndGetFsPathField(response, objectId);

		Path path = FileSystems.getDefault().getPath(pathStr);

		if (Files.notExists(path))

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found in the assigned path " + path);

		try {

			Files.delete(path);

		} catch (IOException e) {

			throw new ObjectStoreServiceException("An error occurs on delete file ", e);

		}

		mongoMetadata.deleteOne(query);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getObject(java.lang.String)

	 */

	@Override

	public String getObject(final String recordId) throws ObjectStoreServiceException {

		Bson query = Filters.eq("id", recordId);

		DBObject response = mongoMetadata.find(query).first();

		if (response == null || !response.containsField(URI_FIELD))

			return null;

		return (String) response.get(URI_FIELD);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String)

	 */

	@Override

	public boolean existIDStartsWith(final String startId) throws ObjectStoreServiceException {

		Bson query = Filters.regex("id", Pattern.compile(startId));

		return mongoMetadata.count(query) > 0;

	}

	@Override

	public boolean dropContent() throws ObjectStoreServiceException {

		if (getBasePath() == null) {

			throw new ObjectStoreServiceException("Error on dropping object store base_path required");

		}

		final Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());

		try {

			FileSystemUtility.deleteFolderRecursive(baseDirPath);

		} catch (IOException e) {

			throw new ObjectStoreServiceException("Error on dropping store ", e);

		}

		log.info("Deleted folder" + baseDirPath.toString());

		if (!Files.exists(baseDirPath)) {

			log.info("Recreating folder " + baseDirPath);

			try {

				Files.createDirectory(baseDirPath);

			} catch (IOException e) {

				throw new ObjectStoreServiceException("Error on dropping store ", e);

			}

		}

		final DeleteResult deleteResult = this.mongoMetadata.deleteMany(new BasicDBObject());

		log.info("Dropped content for object store " + id + ". " + deleteResult.getDeletedCount() + " object(s) deleted.");

		return true;

	}

	@Override

	public String toString() {

		return "FileSystemObjectStore{" +

				"id='" + getId() + '\'' +

				", interpretation='" + getInterpretation() + '\'' +

				", basePath='" + getBasePath() + '\'' +

				", baseURI='" + getBaseURI() + '\'' +

				'}';

	}

	/**

	 * Gets the base uri.

	 *

	 * @return the baseURI

	 */

	public String getBaseURI() {

		return baseURI;

	}

	public String getBasePath() {

		return basePath;

	}

}

/**

 *

 */

package eu.dnetlib.data.objectstore.filesystem;

import eu.dnetlib.miscutils.collections.Pair;

import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.nio.file.*;

import java.nio.file.attribute.BasicFileAttributes;

/**

 * The Class FileSystemUtility.

 *

 * @author sandro

 */

public class FileSystemUtility {

	private static final Log log = LogFactory.getLog(FileSystemUtility.class); // NOPMD by marko on 11/24/08 5:02 PM

	public static Pair<String, Long> saveAndGenerateMD5(final InputStream inputStream, final Path filePath) {

		if(inputStream==null) return null;

		String md5 = null;

		long size = 0;

		try {

			Files.copy(inputStream, filePath);

			FileInputStream fis = new FileInputStream(filePath.toFile());

			md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(fis);

			fis.close();

			size = Files.size(filePath);

		} catch (IOException e1) {

			log.error(e1);

		}

		return new Pair<String, Long>(md5, size);

	}

	/**

	 * Delete folder recursive.

	 *

	 * @param path the path

	 * @return true, if successful

	 * @throws IOException Signals that an I/O exception has occurred.

	 */

	public static boolean deleteFolderRecursive(final Path path) throws IOException {

		Files.walkFileTree(path, new SimpleFileVisitor<Path>() {

			@Override

			public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {

				Files.delete(file);

				return FileVisitResult.CONTINUE;

			}

			@Override

			public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {

				Files.delete(dir);

				return FileVisitResult.CONTINUE;

			}

		});

		return true;

	}

	public static Path objectStoreFilePath(final String basePath, final String objectStoreId, final String objectIdentifier) {

		final Path baseDirPath = FileSystems.getDefault().getPath(basePath).resolve(objectStoreId);

		final String md5id = DnetXsltFunctions.md5(objectIdentifier);

		final String firstLevel = StringUtils.substring(md5id, 0, 2);

		final String secondLevel = StringUtils.substring(md5id, 2, 4);

		final String fileName = StringUtils.substring(md5id, 4) + ".obj";

		return baseDirPath.resolve(firstLevel).resolve(secondLevel).resolve(fileName);

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import java.util.List;

import com.google.common.collect.Lists;

import com.mongodb.DBObject;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoCursor;

import com.mongodb.client.model.Filters;

import com.mongodb.client.model.Sorts;

import eu.dnetlib.enabling.resultset.ResultSet;

import eu.dnetlib.enabling.resultset.ResultSetAware;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.MappedCollection;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.conversions.Bson;

/**

 * The listener interface for receiving fileSystemObjectStoreResultSet events.

 * The class that is interested in processing a fileSystemObjectStoreResultSet

 * event implements this interface, and the object created

 * with that class is registered with a component using the

 * component's <code>addFileSystemObjectStoreResultSetListener<code> method. When

 * the fileSystemObjectStoreResultSet event occurs, that object's appropriate

 * method is invoked.

 *

 * @author sandro

 */

public class FileSystemObjectStoreResultSetListener implements ResultSetListener, ResultSetAware {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(FileSystemObjectStoreResultSetListener.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The from date. */

	private Long fromDate;

	/** The until date. */

	private Long untilDate;

	/** The records. */

	private List<String> records;

	/** The object store id. */

	private String objectStoreID;

	/** The mongo collection. */

	private MongoCollection<DBObject> mongoCollection;

	/** The base uri. */

	private String baseURI;

	/**

	 * The base path

	 */

	private String basePath;

	/** The current size. */

	private int currentSize = -1;

	/** The current cursor. */

	private MongoCursor<DBObject> currentCursor;

	/** The cursor position. */

	private long cursorPosition;

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)

	 */

	@Override

	public List<String> getResult(final int from, final int to) {

		if (log.isDebugEnabled()) {

			log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));

		}

		if (records != null) {

			List<String> ids = Lists.newArrayList();

			for (int i = from-1; i < Math.min(records.size(),to); i++) {

				ids.add(records.get(i));

			}

			Bson q = Filters.in("id", ids);

			FindIterable<DBObject> res = getMongoCollection().find(q);

			return MappedCollection.listMap(res, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));

		} else if ((fromDate != null) && (untilDate != null)) {

			if ((currentCursor == null) || (cursorPosition > from)) {

				createCurrentCursor();

			}

			while (cursorPosition < from) {

				currentCursor.next();

				cursorPosition++;

			}

			List<DBObject> result = Lists.newArrayList();

			for (int i = from; i <= to; i++) {

				if (currentCursor.hasNext()) {

					result.add(currentCursor.next());

					cursorPosition++;

				}

			}

			return MappedCollection.listMap(result, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));

		}

		throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");

	}

	/**

	 * Creates the current cursor.

	 */

	private void createCurrentCursor() {

		Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));

		if (currentCursor != null) {

			currentCursor.close();

		}

		currentCursor = getMongoCollection().find(timestampQuery).sort(Sorts.orderBy(Filters.eq("_id", 1))).iterator();

		cursorPosition = 1;

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()

	 */

	@Override

	public int getSize() {

		if (currentSize == -1) {

			currentSize = calculateSize();

		}

		return Math.max(0, currentSize - 1);

	}

	/**

	 * Calculate size.

	 *

	 * @return the int

	 */

	private int calculateSize() {

		if (records != null) {

			Bson query = Filters.in("id", records);

			return (int) getMongoCollection().count(query);

		} else if ((fromDate != null) && (untilDate != null)) {

			Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));

			return (int) getMongoCollection().count(timestampQuery);

		}

		return 0;

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)

	 */

	@Override

	public void setResultSet(final ResultSet resultSet) {

		resultSet.close();

	}

	/**

	 * Gets the from date.

	 *

	 * @return the fromDate

	 */

	public Long getFromDate() {

		return fromDate;

	}

	/**

	 * Sets the from date.

	 *

	 * @param fromDate the fromDate to set

	 */

	public FileSystemObjectStoreResultSetListener setFromDate(final Long fromDate) {

		this.fromDate = fromDate;

		return this;

	}

	/**

	 * Gets the until date.

	 *

	 * @return the untilDate

	 */

	public Long getUntilDate() {

		return untilDate;

	}

	/**

	 * Sets the until date.

	 *

	 * @param untilDate the untilDate to set

	 */

	public FileSystemObjectStoreResultSetListener setUntilDate(final Long untilDate) {

		this.untilDate = untilDate;

		return this;

	}

	/**

	 * Gets the records.

	 *

	 * @return the records

	 */

	public List<String> getRecords() {

		return records;

	}

	/**

	 * Sets the records.

	 *

	 * @param records the records to set

	 */

	public void setRecords(final List<String> records) {

		this.records = records;

	}

	/**

	 * Gets the object store id.

	 *

	 * @return the objectStoreID

	 */

	public String getObjectStoreID() {

		return objectStoreID;

	}

	/**

	 * Sets the object store id.

	 *

	 * @param objectStoreID the objectStoreID to set

	 */

	public void setObjectStoreID(final String objectStoreID) {

		this.objectStoreID = objectStoreID;

	}

	/**

	 * Gets the base uri.

	 *

	 * @return the baseURI

	 */

	public String getBaseURI() {

		return baseURI;

	}

	/**

	 * Sets the base uri.

	 *

	 * @param baseURI the baseURI to set

	 */

	public void setBaseURI(final String baseURI) {

		this.baseURI = baseURI;

	}

	/**

	 * Gets the current size.

	 *

	 * @return the currentSize

	 */

	public int getCurrentSize() {

		return currentSize;

	}

	/**

	 * Sets the current size.

	 *

	 * @param currentSize the currentSize to set

	 */

	public void setCurrentSize(final int currentSize) {

		this.currentSize = currentSize;

	}

	/**

	 * Gets the current cursor.

	 *

	 * @return the currentCursor

	 */

	public MongoCursor<DBObject> getCurrentCursor() {

		return currentCursor;

	}

	/**

	 * Sets the current cursor.

	 *

	 * @param currentCursor the currentCursor to set

	 */

	public void setCurrentCursor(final MongoCursor<DBObject> currentCursor) {

		this.currentCursor = currentCursor;

	}

	/**

	 * Gets the cursor position.

	 *

	 * @return the cursorPosition

	 */

	public long getCursorPosition() {

		return cursorPosition;

	}

	/**

	 * Sets the cursor position.

	 *

	 * @param cursorPosition the cursorPosition to set

	 */

	public void setCursorPosition(final long cursorPosition) {

		this.cursorPosition = cursorPosition;

	}

	/**

	 * Gets the mongo collection.

	 *

	 * @return the mongo collection

	 */

	public MongoCollection<DBObject> getMongoCollection() {

		return mongoCollection;

	}

	/**

	 * Sets the mongo collection.

	 *

	 * @param mongoCollection the new mongo collection

	 */

	public void setMongoCollection(final MongoCollection<DBObject> mongoCollection) {

		this.mongoCollection = mongoCollection;

	}

	public String getBasePath() {

		return basePath;

	}

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

}

package eu.dnetlib.data.objectstore.filesystem;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.IndexOptions;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_ID_FIELD;

import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_METADATA_NAME_FIELD;

/**

 * Created by claudio on 15/09/16.

 */

public class IndexIntegrityChecker {

	private static final Log log = LogFactory.getLog(IndexIntegrityChecker.class);

	@Autowired

	private ObjectStoreDao dao;

	public void check() {

		checkIndexes();

	}

	private void checkIndexes() {

		log.info("objectStore indexes integrity start");

		final MongoDatabase db = ((FileSystemObjectStoreDao) dao).getDb();

		final IndexOptions bg = new IndexOptions().background(true);

		for (String objectStoreId : dao.listObjectStores()) {

			final String id = StringUtils.substringBefore(objectStoreId, "_");

			final MongoCollection<DBObject> objectStore = db.getCollection(id, DBObject.class);

			if (log.isDebugEnabled()) {

				log.debug(String.format("creating index (id, timestamp) on objectStore %s", id));

			}

			objectStore.createIndex(new BasicDBObject("id", 1), bg);

			objectStore.createIndex(new BasicDBObject("timestamp", 1), bg);

		}

		if (log.isDebugEnabled()) {

			log.debug(String.format("creating index (%s) on %s", OBJECTSTORE_ID_FIELD, OBJECTSTORE_METADATA_NAME_FIELD));

		}

		final MongoCollection<DBObject> metadata = db.getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

		metadata.createIndex(new BasicDBObject(OBJECTSTORE_ID_FIELD, 1), bg);

		log.info("objectStore indexes integrity completed");

	}

}

/**

 *

 */

package eu.dnetlib.data.objectstore.filesystem;

import java.io.IOException;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.List;

import javax.annotation.Resource;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.Filters;

import com.mongodb.client.result.UpdateResult;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.miscutils.collections.MappedCollection;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.conversions.Bson;

/**

 * @author sandro

 *

 */

public class FileSystemObjectStoreDao implements ObjectStoreDao {

	public static final String INTERPRETATION_FIELD = "interpretation";

	public final static String OBJECTSTORE_METADATA_NAME_FIELD = "metadataObjectStore";

	public final static String OBJECTSTORE_ID_FIELD = "obsId";

	public final static String BASE_PATH_FIELD = "basePath";

	private static final Log log = LogFactory.getLog(FileSystemObjectStoreDao.class); // NOPMD by marko on 11/24/08 5:02 PM

	private static final String OBJECTSTORE_PROFILE_SUFFIX = "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl";

	@Resource(name="objectstoreMongoDB")

	private MongoDatabase db;

	private boolean upsert;

	private String objectStoreRESTURI;

	/**

	 * {@inheritDoc}

	 * @throws ObjectStoreServiceException

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#getObjectStore(java.lang.String)

	 */

	@Override

	public ObjectStore getObjectStore(final String obsId) throws ObjectStoreServiceException {

		String currentId = obsId.substring(0, 36);

		String find_id = obsId;

		if (find_id.length() == 36) {

			find_id += OBJECTSTORE_PROFILE_SUFFIX;

		}

		MongoCollection<DBObject> metadataObjectStore = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, find_id);

		log.debug("QUERY :" + query.toString());

		DBObject resultQuery = metadataObjectStore.find(query).first();

		log.debug("result " + resultQuery);

		if ((resultQuery == null)) throw new ObjectStoreFileNotFoundException("the objectStore with identifier: "+obsId+" was not found");

		final String basePath = resultQuery.get(BASE_PATH_FIELD).toString();

		final String interpretation = resultQuery.get("interpretation").toString();

		if (!resultQuery.containsField(BASE_PATH_FIELD) || StringUtils.isBlank(basePath))

			throw new ObjectStoreServiceException("Can't Get Objectstore, the metadata doesn't contain info about the basepath");

		final MongoCollection<DBObject> collection = getDb().getCollection(currentId, DBObject.class);

		return new FileSystemObjectStore(currentId, interpretation, basePath, collection, objectStoreRESTURI);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#listObjectStores()

	 */

	@Override

	public List<String> listObjectStores() {

		MongoCollection<DBObject> metadata = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

		return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() {

			@Override

			public String evaluate(final DBObject object) {

				return (String) object.get(OBJECTSTORE_ID_FIELD);

			}

		});

	}

	/**

	 * {@inheritDoc}

	 *

	 * @throws ObjectStoreServiceException

	 */

	@Override

	public boolean createObjectStore(final String obsId, final String interpretation, final String basePath) throws ObjectStoreServiceException {

		log.debug(String.format("Create object Store method\n\t Id:%s  Interpretation:%s BasePath : %s", obsId, interpretation, basePath) );

		if (StringUtils.isBlank(basePath)) throw new ObjectStoreServiceException("Can't create the object store: the base path cannot be blank");

		Path path = FileSystems.getDefault().getPath(basePath);

		if (!Files.exists(path) || !Files.isDirectory(path))

			throw new ObjectStoreServiceException("Can't create the object store: base path: '" + basePath + "' doesn't exist or it is not a folder");

		try {

			String currentObsId = obsId.substring(0, 36);

			log.debug("Cleaned objectStore Id " + currentObsId);

			if (Files.exists(path.resolve(currentObsId)))

				throw new ObjectStoreServiceException("Can't create the object store: base path: '" + path.resolve(currentObsId) + "' already exists");

			Files.createDirectory(path.resolve(currentObsId));

			MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

			final BasicDBObject obj = new BasicDBObject();

			obj.put(OBJECTSTORE_ID_FIELD, obsId);

			obj.put(INTERPRETATION_FIELD, interpretation);

			obj.put(BASE_PATH_FIELD, basePath);

			coll.insertOne(obj);

			MongoCollection<DBObject> objectStore = getDb().getCollection(currentObsId, DBObject.class);

			objectStore.createIndex(new BasicDBObject("id", 1));

			objectStore.createIndex(new BasicDBObject("timestamp", 1));

			return true;

		} catch (Throwable e) {

			throw new ObjectStoreServiceException("Can't Create the object Store id: '" + obsId, e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#updateObjectStore(java.lang.String, java.lang.String)

	 */

	@Override

	public boolean updateObjectStore(final String obsId, final String interpretation) {

		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

		final BasicDBObject obj = new BasicDBObject();

		obj.put("$set", new BasicDBObject(INTERPRETATION_FIELD, interpretation));

		final UpdateResult updateResult = coll.updateOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId), obj);

		if (updateResult.isModifiedCountAvailable()) {

			log.debug("Matched / Modified " + updateResult.getMatchedCount() + " / " + updateResult.getModifiedCount());

		}

		return true;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @throws ObjectStoreServiceException

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#deleteObjectStore(java.lang.String)

	 */

	@Override

	public boolean deleteObjectStore(final String obsId) throws ObjectStoreServiceException {

		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);

		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, obsId);

		DBObject resultQuery = coll.find(query).first();

		String basePath = checkAndGetFsPathField(resultQuery, obsId);

		String currentObsId = obsId.substring(0, 36);

		Path basePathFS = FileSystems.getDefault().getPath(basePath, currentObsId);

		if (!Files.exists(basePathFS))

			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId + ": the base path does not exist :" + basePathFS);

		try {

			FileSystemUtility.deleteFolderRecursive(basePathFS);

		} catch (IOException e) {

			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId, e);

		}

		coll.deleteOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId));