D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-gridfs-objectstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-gridfs-objectstore"}

package eu.dnetlib.test;

import com.google.common.collect.Lists;

import com.mongodb.*;

import com.mongodb.gridfs.GridFS;

import eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl;

import org.junit.Ignore;

import org.junit.Test;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

import java.net.UnknownHostException;

import java.util.ArrayList;

import java.util.Date;

public class TestQuery {

	@Test

	@Ignore

	public void testFromUntil() throws UnknownHostException, MongoException, UnsupportedEncodingException {

		GridFSObjectstoreDaoImpl dao = new GridFSObjectstoreDaoImpl();

		Mongo mongo = new Mongo("localhost", 27017);

		DB db = mongo.getDB("objectStore");

		dao.setDb(db);

		GridFS obs = new GridFS(db, "aeiouy");

		BasicDBObject q = new BasicDBObject();

		Date d = new Date();

		Double from = new Double(d.getTime());

		System.out.println(URLEncoder.encode("MY CRZY QUERY! +&+ :)", "UTF-8"));

		q.put("$lt", from.doubleValue());

		DBCursor files = obs.getFileList(new BasicDBObject("metadata.timestamp", q)).sort(new BasicDBObject("metadata.id", 1)).skip(4).limit(3);

		ArrayList<DBObject> p = Lists.newArrayList((Iterable<DBObject>) files);

		for (DBObject tmp : p) {

			DBObject o = (DBObject) tmp.get("metadata");

			System.out.println(o);

		}

	}

}

package eu.dnetlib.test;

import java.net.UnknownHostException;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

import com.mongodb.DB;

import com.mongodb.MongoClient;

@Configuration

public class ConfigurationTestConfig {

	@Bean

	public DB db() throws UnknownHostException {

		MongoClient mongo = new MongoClient();

		return mongo.getDB("objectStore");

	}

}

package eu.dnetlib.test;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import com.mongodb.DB;

import com.mongodb.gridfs.GridFS;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectStore;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class TestGridfs {

	@Autowired

	private DB db;

	@Ignore

	@Test

	public void testTag() {

		final GridFS collection = new GridFS(db, "testSTORE");

		ObjectStore objectStore = new GridFSObjectStore("85499fec-7ef8-4bae-8b51-6eb1abc12076", collection, true);

		final double start = System.currentTimeMillis();

		objectStore.untagObjectstore("suca");

		final double end = System.currentTimeMillis();

		System.out.println("Done in end = " + (end - start) / 1000);

	}

	@Ignore

	@Test

	public void populationTest() throws FileNotFoundException, ObjectStoreServiceException {

		final GridFS collection = new GridFS(db, "testSTORE");

		ObjectStore objectStore = new GridFSObjectStore("85499fec-7ef8-4bae-8b51-6eb1abc12076", collection, true);

		for (int i = 300000; i < 1000000; i++) {

			final ObjectStoreRecord record;

			record = new ObjectStoreRecord();

			record.setInputStream(new FileInputStream(new File("/Users/sandro/dropall.js")));

			final ObjectStoreFile metadata = new ObjectStoreFile();

			metadata.setObjectID("" + i);

			metadata.setAccessProtocol(Protocols.File_System);

			metadata.setDownloadedURL("suca");

			metadata.setMimeType("javascript");

			record.setFileMetadata(metadata);

			objectStore.feedObjectRecord(record);

		}

	}

}

package eu.dnetlib.data.objectstore.modular.gridFS;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

import java.util.List;

import java.util.Set;

import java.util.regex.Pattern;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import com.mongodb.*;

import com.mongodb.gridfs.GridFS;

import com.mongodb.gridfs.GridFSDBFile;

import com.mongodb.gridfs.GridFSInputFile;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.Pair;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class GridFSObjectStore.

 */

public class GridFSObjectStore implements ObjectStore {

	// ================================================================================

	// Properties

	// ================================================================================

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(GridFSObjectStore.class);

	/**

	 * The collection.

	 */

	private final GridFS collection;

	/** The id. */

	private String id;

	/** The upsert. */

	private boolean upsert;

	/** The base uri. */

	private String baseURI;

	/** The interpretation. */

	private String interpretation;

	// ================================================================================

	// Constructor

	// ================================================================================

	/**

	 * Instantiates a new gridfs objectstore.

	 *

	 * @param id

	 *            the id

	 * @param collection

	 *            the collection

	 * @param upsert

	 *            the upsert

	 */

	public GridFSObjectStore(final String id, final GridFS collection, final boolean upsert) {

		this.id = id;

		this.setUpsert(upsert);

		this.collection = collection;

	}

	// ================================================================================

	// Getter and Setter

	// ================================================================================

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId()

	 */

	/**

	 * Gets the id.

	 *

	 * @return the id

	 */

	@Override

	public String getId() {

		return id;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize()

	 */

	/**

	 * Gets the size.

	 *

	 * @return the size

	 */

	@Override

	public int getSize() {

		return collection.getFileList().count();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation()

	 */

	/**

	 * Gets the interpretation.

	 *

	 * @return the interpretation

	 */

	@Override

	public String getInterpretation() {

		if (interpretation == null) {

			this.interpretation = (String) getMDStoreMetadata().get("interpretation");

		}

		return this.interpretation;

	}

	/**

	 * Gets the MD store metadata.

	 *

	 * @return the MD store metadata

	 */

	public DBObject getMDStoreMetadata() {

		return collection.getDB().getCollection("metadataObjectStore").findOne(new BasicDBObject("obsId", Pattern.compile(getId())));

	}

	/**

	 * Gets the base uri.

	 *

	 * @return the base uri

	 */

	public String getBaseURI() {

		return baseURI;

	}

	/**

	 * Sets the base uri.

	 *

	 * @param baseURI

	 *            the new base uri

	 */

	public void setBaseURI(final String baseURI) {

		this.baseURI = baseURI;

	}

	/**

	 * Checks if is upsert.

	 *

	 * @return true, if is upsert

	 */

	public boolean isUpsert() {

		return upsert;

	}

	/**

	 * Sets the upsert.

	 *

	 * @param upsert

	 *            the new upsert

	 */

	public void setUpsert(final boolean upsert) {

		this.upsert = upsert;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String)

	 */

	/**

	 * Delete object.

	 *

	 * @param objectId

	 *            the object id

	 */

	@Override

	public void deleteObject(final String objectId) {

		collection.remove(new BasicDBObject("_id", objectId));

	}

	// ================================================================================

	// Deliver

	// ================================================================================

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Double, java.lang.Double)

	 */

	/**

	 * Deliver.

	 *

	 * @param from

	 *            the from

	 * @param until

	 *            the until

	 * @return the result set listener

	 */

	@Override

	public ResultSetListener deliver(final Long from, final Long until) {

		final GridFSObjectstoreResultSetListener resulset = new GridFSObjectstoreResultSetListener();

		resulset.setBaseURI(baseURI);

		resulset.setObjectStoreID(id);

		resulset.setCollection(collection);

		resulset.setFromDate(from);

		resulset.setUntilDate(until);

		return resulset;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable)

	 */

	/**

	 * Deliver ids.

	 *

	 * @param ids

	 *            the ids

	 * @return the result set listener

	 */

	@Override

	public ResultSetListener deliverIds(final Iterable<String> ids) {

		final GridFSObjectstoreResultSetListener resulset = new GridFSObjectstoreResultSetListener();

		resulset.setBaseURI(baseURI);

		resulset.setObjectStoreID(id);

		resulset.setCollection(collection);

		resulset.setRecords((List<String>) ids);

		return resulset;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String)

	 */

	/**

	 * Deliver object.

	 *

	 * @param objectId

	 *            the object id

	 * @return the object store file

	 * @throws ObjectStoreServiceException

	 *             the object store service exception

	 */

	@Override

	public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {

		final BasicDBObject query = new BasicDBObject("_id", objectId);

		final List<GridFSDBFile> file = collection.find(query);

		checkSingleItem(objectId, file);

		return ObjectStoreFileUtility.build(file.get(0), baseURI, id);

	}

	/**

	 * Deliver records by tag.

	 *

	 * @param tags

	 *            the tags

	 * @return the string

	 */

	@Override

	public ResultSetListener deliverRecordsByTag(final Iterable<String> tags) {

		final GridFSObjectstoreResultSetListener resulset = new GridFSObjectstoreResultSetListener();

		resulset.setBaseURI(baseURI);

		resulset.setObjectStoreID(id);

		resulset.setCollection(collection);

		resulset.setTags(Sets.newHashSet(tags));

		return resulset;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverStream(java.lang.String)

	 */

	/**

	 * Deliver stream.

	 *

	 * @param objectId

	 *            the object id

	 * @return the input stream

	 * @throws ObjectStoreFileNotFoundException

	 *             the object store file not found exception

	 */

	@Override

	public InputStream deliverStream(final String objectId) throws ObjectStoreFileNotFoundException {

		final BasicDBObject query = new BasicDBObject("_id", objectId);

		final List<GridFSDBFile> file = collection.find(query);

		checkSingleItem(objectId, file);

		return file.get(0).getInputStream();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String)

	 */

	/**

	 * Exist id starts with.

	 *

	 * @param startId

	 *            the start id

	 * @return true, if successful

	 */

	@Override

	public boolean existIDStartsWith(final String startId) {

		final BasicDBObject query = new BasicDBObject("_id", startId);

		final List<GridFSDBFile> file = collection.find(query);

		return file.size() > 0;

	}

	// ================================================================================

	// Feed

	// ================================================================================

	/*

	 * (non-Javadoc)

	 *

	 * @see

	 * eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)

	 */

	/**

	 * Feed object record.

	 *

	 * @param record

	 *            the record

	 * @return the string

	 * @throws ObjectStoreServiceException

	 *             the object store service exception

	 */

	@Override

	public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {

		if (record == null || record.getFileMetadata() == null) throw new ObjectStoreServiceException("Empty input Record");

		if (existIDStartsWith(record.getFileMetadata().getObjectID())) {

			log.debug("Object already exist ");

			if (record.getInputStream() != null) {

				try {

					record.getInputStream().close();

				} catch (IOException e) {

					log.debug("Exception happen in closing inputstream " + e);

					throw new ObjectStoreServiceException(e);

				}

			}

			ObjectStoreFile obj = deliverObject(record.getFileMetadata().getObjectID());

			return obj.getURI();

		}

		long timestamp = System.currentTimeMillis();

		String URI = "";

		if (record.getInputStream() == null) throw new ObjectStoreServiceException("missing inputstream on record " + record.getFileMetadata().getObjectID());

		final GridFSInputFile currentFile = collection.createFile(record.getInputStream());

		currentFile.setId(record.getFileMetadata().getObjectID());

		currentFile.setFilename(record.getFileMetadata().getObjectID());

		final BasicDBObject metadata = new BasicDBObject();

		metadata.put("id", record.getFileMetadata().getObjectID());

		metadata.put("mime", record.getFileMetadata().getMimeType());

		metadata.put("originalObject", record.getFileMetadata().toJSON());

		metadata.put("timestamp", timestamp);

		try {

			URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId="

					+ URLEncoder.encode(record.getFileMetadata().getObjectID(), "UTF-8");

			metadata.put("uri", URI);

			if (metadata.get("tags") == null) {

				metadata.put("tags", Sets.newHashSet("UNVISITED"));

			} else {

				final BasicDBList tagList = (BasicDBList) metadata.get("tags");

				final Set<String> inputSet = Sets.newHashSet();

				for (final Object o : tagList) {

					inputSet.add((String) o);

				}

				inputSet.add("UNVISITED");

				metadata.put("tags", inputSet);

			}

			currentFile.setMetaData(metadata);

			currentFile.save();

		} catch (Exception e) {

			log.error(e);

			throw new ObjectStoreServiceException(e);

		} finally {

			try {

				record.getInputStream().close();

			} catch (IOException e) {

				log.error(e);

			}

		}

		return URI;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)

	 */

	/**

	 * Feed metadata record.

	 *

	 * @param records

	 *            the records

	 * @param incremental

	 *            the incremental

	 * @return the int

	 */

	@Override

	public int feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) {

		final long timestamp = System.currentTimeMillis();

		for (final MetadataObjectRecord o : records) {

			if (o == null || o.getRecord() == null) {

				log.debug("Null object metadata record");

				continue;

			}

			final GridFSInputFile currentFile = collection.createFile(new ByteArrayInputStream(o.getRecord().getBytes()));

			currentFile.setId(o.getId());

			final BasicDBObject metadata = new BasicDBObject();

			metadata.put("id", o.getId());

			metadata.put("mime", o.getMime());

			metadata.put("timestamp", timestamp);

			try {

				final String URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId=" + URLEncoder.encode(o.getId(), "UTF-8");

				metadata.put("uri", URI);

			} catch (UnsupportedEncodingException e) {

				log.error("Got an exception during the feed ", e);

			}

			currentFile.setMetaData(metadata);

			currentFile.save();

		}

		return this.getSize();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean)

	 */

	/**

	 * Feed.

	 *

	 * @param records

	 *            the records

	 * @param incremental

	 *            the incremental

	 * @return the int

	 */

	@Override

	public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) {

		final long timestamp = System.currentTimeMillis();

		for (final ObjectStoreRecord o : records) {

			if (o == null || o.getInputStream() == null) {

				if (o != null) {

					log.debug("Null object " + o.getFileMetadata().toJSON());

				} else {

					log.debug("Null Object");

				}

				try {

					Thread.sleep(1000);

				} catch (InterruptedException e) {

					log.error(e);

				}

				continue;

			}

			if (alreadyExist(o.getFileMetadata().getObjectID())) {

				try {

					o.getInputStream().close();

				} catch (IOException e) {

					log.error("Error on closing inputStream ", e);

				}

				continue;

			}

			final GridFSInputFile currentFile = collection.createFile(o.getInputStream());

			currentFile.setId(o.getFileMetadata().getObjectID());

			currentFile.setFilename(o.getFileMetadata().getObjectID());

			final BasicDBObject metadata = new BasicDBObject();

			metadata.put("id", o.getFileMetadata().getObjectID());

			metadata.put("mime", o.getFileMetadata().getMimeType());

			metadata.put("originalObject", o.getFileMetadata().toJSON());

			metadata.put("timestamp", timestamp);

			try {

				final String URI = baseURI + "?objectStore=" + URLEncoder.encode(id, "UTF-8") + "&objectId="

						+ URLEncoder.encode(o.getFileMetadata().getObjectID(), "UTF-8");

				metadata.put("uri", URI);

			} catch (UnsupportedEncodingException e) {

				log.error("Got an exception during the feed ", e);

			}

			currentFile.setMetaData(metadata);

			currentFile.save();

		}

		return getSize();

	}

	// ================================================================================

	// Tag Section

	// ================================================================================

	/**

	 * Tag ids.

	 *

	 * @param ids

	 *            the ids

	 * @param tag

	 *            the tag

	 * @return the int

	 */

	@Override

	public int tagIds(final Iterable<String> ids, final String tag) {

		if (ids == null) return 0;

		int counter = 0;

		for (final String id : ids) {

			DBObject query = BasicDBObjectBuilder.start().add("_id", id).get();

			final GridFSDBFile foundObject = collection.findOne(query);

			final DBObject metadata = foundObject.getMetaData();

			final DBObject currentTags = (DBObject) metadata.get("tags");

			counter++;

			if (currentTags == null) {

				metadata.put("tags", tag);

			} else {

				final BasicDBList tagList = (BasicDBList) currentTags;

				final Set<String> inputSet = Sets.newHashSet();

				for (final Object o : tagList) {

					inputSet.add((String) o);

				}

				inputSet.add(tag);

				metadata.put("tags", inputSet);

			}

			foundObject.save();

		}

		return counter;

	}

	/**

	 * Untag ids.

	 *

	 * @param ids

	 *            the ids

	 * @param tag

	 *            the tag

	 * @return the int

	 */

	@Override

	public int untagIds(final Iterable<String> ids, final String tag) {

		if (ids == null) return 0;

		int counter = 0;

		for (final String id : ids) {

			DBObject query = BasicDBObjectBuilder.start().add("_id", id).get();

			final GridFSDBFile foundObject = collection.findOne(query);

			final DBObject metadata = foundObject.getMetaData();

			final DBObject currentTags = (DBObject) metadata.get("tags");

			if (currentTags != null) {

				counter++;

				final BasicDBList tagList = (BasicDBList) currentTags;

				final Set<String> inputSet = Sets.newHashSet();

				for (final Object o : tagList) {

					inputSet.add((String) o);

				}

				inputSet.remove(tag);

				metadata.put("tags", inputSet);

			}

			foundObject.save();

		}

		return counter;

	}

	/**

	 * Tag objectstore.

	 *

	 * @param tag

	 *            the tag

	 * @return the int

	 */

	@Override

	public int tagObjectstore(final String tag) {

		log.info(String.format("Starting to tag (%s) for object store %s" , tag, id));

		final BasicDBObject fields = new BasicDBObject("metadata.tags", tag);

		final BasicDBObject update = new BasicDBObject("$push", fields);

		final DBCollection mongoCollection = collection.getDB().getCollection(collection.getBucketName() + ".files");

		final WriteResult result = mongoCollection.update(new BasicDBObject(), update, true, true);

		return result.getN();

	}

	/**

	 * Untag objectstore.

	 *

	 * @param tag

	 *            the tag

	 * @return the int

	 */

	@Override

	public int untagObjectstore(final String tag) {

		log.info(String.format("Starting to untag (%s) for object store %s" , tag, id));

		final BasicDBList objects = new BasicDBList();

		objects.add(tag);

		final BasicDBObject fields = new BasicDBObject("metadata.tags", new BasicDBObject("$in", objects));

		final BasicDBObject update = new BasicDBObject("$pull", fields);

		DBCollection mongoCollection = collection.getDB().getCollection(collection.getBucketName() + ".files");

		final WriteResult result = mongoCollection.update(new BasicDBObject(), update, true, true);

		return result.getN();

	}

	// ================================================================================

	// Private Method

	// ================================================================================

	/**

	 * Check single item.

	 *

	 *

	 * @param objectId

	 *            the object id

	 * @param file

	 *            the file

	 * @throws ObjectStoreFileNotFoundException

	 *             the object store file not found exception

	 */

	private void checkSingleItem(final String objectId, final List<GridFSDBFile> file) throws ObjectStoreFileNotFoundException {

		if (file.isEmpty()) throw new ObjectStoreFileNotFoundException(String.format("Object file not found with id: %s", objectId));

		if (file.size() > 1) throw new IllegalStateException(String.format("More than one objects found with id: %s", objectId));

	}

	/**

	 * Already exist.

	 *

	 * @param objectId

	 *            the object id

	 * @return true, if successful

	 */

	private boolean alreadyExist(final String objectId) {

		final BasicDBObject query = new BasicDBObject("_id", objectId);

		final List<GridFSDBFile> file = collection.find(query);

		return file != null && file.size() > 0;

	}

	@Override

	public List<Pair<String, Integer>> listTags() {

		String map = "function () {" + "this.metadata.tags.forEach(function(data) { emit (data, 1)  } );" + "}";

		String reduce = "function (key, values) {  return Array.sum(values) } ";

		DBCollection inputCollection = collection.getDB().getCollection(this.id + ".files");

		MapReduceCommand cmd = new MapReduceCommand(inputCollection, map, reduce, "", MapReduceCommand.OutputType.INLINE, QueryBuilder.start().get());

		MapReduceOutput out = inputCollection.mapReduce(cmd);

		List<Pair<String, Integer>> tags = Lists.newArrayList();

		for (DBObject o : out.results()) {

			String tag_id = (String) o.get("_id");

			Double tag_count = (Double) o.get("value");

			tags.add(new Pair<String, Integer>(tag_id, tag_count.intValue()));

		}

		return tags;

	}

}

package eu.dnetlib.data.objectstore.modular.gridFS;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.FactoryBean;

import com.mongodb.MongoOptions;

public class MongoGridFSOptionsFactory implements FactoryBean<MongoOptions> {

	private int connectionsPerHost;

	@Override

	public MongoOptions getObject() throws BeansException {

		MongoOptions opts = new MongoOptions();

		opts.connectionsPerHost = connectionsPerHost;

		return opts;

	}

	@Override

	public Class<MongoOptions> getObjectType() {

		return MongoOptions.class;

	}

	@Override

	public boolean isSingleton() {

		return false;

	}

	public int getConnectionsPerHost() {

		return connectionsPerHost;

	}

	public void setConnectionsPerHost(final int connectionsPerHost) {

		this.connectionsPerHost = connectionsPerHost;

	}

}

package eu.dnetlib.data.objectstore.modular.gridFS;

import java.util.ArrayList;

import java.util.List;

import java.util.Set;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.DBCursor;

import com.mongodb.DBObject;

import com.mongodb.QueryBuilder;

import com.mongodb.gridfs.GridFS;

import com.mongodb.gridfs.GridFSDBFile;

import eu.dnetlib.enabling.resultset.ResultSet;

import eu.dnetlib.enabling.resultset.ResultSetAware;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.MappedCollection;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * The listener interface for receiving gridFSObjectstoreResultSet events. The class that is interested in processing a

 * gridFSObjectstoreResultSet event implements this interface, and the object created with that class is registered with a component using

 * the component's <code>addGridFSObjectstoreResultSetListener<code> method. When the gridFSObjectstoreResultSet event occurs, that object's

 * appropriate method is invoked.

 *

 */

public class GridFSObjectstoreResultSetListener implements ResultSetListener, ResultSetAware {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(GridFSObjectstoreResultSetListener.class);

	/** The from date. */

	private Long fromDate;

	/** The until date. */

	private Long untilDate;

	/** The records. */

	private List<String> records;

	/** The object store id. */

	private String objectStoreID;

	/** The collection. */

	private GridFS collection;

	/** The base uri. */

	private String baseURI;

	/** The current size. */

	private int currentSize = -1;

	/** The current cursor. */

	private DBCursor currentCursor;

	/** The cursor position. */

	private long cursorPosition;

	/** The tags. */

	private Set<String> tags;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)

	 */

	@Override

	public List<String> getResult(final int from, final int to) {

		log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));

		if (records != null) {

			List<String> ids = Lists.newArrayList();

			for (int i = from; i < to; i++) {

				ids.add(records.get(i));

			}

			QueryBuilder qBuilder = QueryBuilder.start("metadata.id").in(ids);

			DBObject q = qBuilder.get();

			List<GridFSDBFile> out = collection.find(q);

			return MappedCollection.listMap(out, ObjectStoreFileUtility.asJSON(baseURI, objectStoreID));

		} else if (tags != null && !tags.isEmpty()) {

			if (currentCursor == null || cursorPosition > from) {

				createCurrentCursor(this.tags);

			}

			while (cursorPosition < from) {

				currentCursor.next();

				cursorPosition++;

			}

			ArrayList<GridFSDBFile> out = new ArrayList<GridFSDBFile>();

			for (int i = from; i <= to; i++) {

				if (currentCursor.hasNext()) {

					out.add((GridFSDBFile) currentCursor.next());

					cursorPosition++;

				}

			}

			return MappedCollection.listMap(out, ObjectStoreFileUtility.asJSON(baseURI, objectStoreID));

		}

		else if (fromDate != null && untilDate != null) {

			if (currentCursor == null || cursorPosition > from) {

				createCurrentCursor();

			}

			while (cursorPosition < from) {

				currentCursor.next();

				cursorPosition++;

			}

			ArrayList<GridFSDBFile> out = new ArrayList<GridFSDBFile>();

			for (int i = from; i <= to; i++) {

				if (currentCursor.hasNext()) {

					out.add((GridFSDBFile) currentCursor.next());

					cursorPosition++;

				}

			}

			return MappedCollection.listMap(out, ObjectStoreFileUtility.asJSON(baseURI, objectStoreID));

		}

		throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");

	}

	/**

	 * Creates the current cursor.

	 */

	private void createCurrentCursor(final Set<String> tags) {

		QueryBuilder qBuilder = QueryBuilder.start("metadata.tags").in(tags);

		if (currentCursor != null) {

			currentCursor.close();

		}

		currentCursor = collection.getFileList(qBuilder.get()).sort(new BasicDBObject("_id", 1));

		currentCursor.batchSize(20);

		cursorPosition = 1;

	}

	/**

	 * Creates the current cursor.

	 */

	private void createCurrentCursor() {

		BasicDBObject query = new BasicDBObject();

		query.put("$gt", fromDate.doubleValue());

		query.put("$lt", untilDate.doubleValue());

		if (currentCursor != null) {

			currentCursor.close();

		}

		currentCursor = collection.getFileList(new BasicDBObject("metadata.timestamp", query)).sort(new BasicDBObject("_id", 1));

		currentCursor.batchSize(20);

		cursorPosition = 1;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()

	 */

	@Override

	public int getSize() {

		if (currentSize == -1) {

			currentSize = calculateSize();

		}

		return currentSize;

	}

	/**

	 * Calculate size.

	 *

	 * @return the int

	 */

	private int calculateSize() {

		if (records != null) {

			QueryBuilder qBuilder = QueryBuilder.start("metadata.id").in(records);

			DBObject q = qBuilder.get();

			List<GridFSDBFile> out = collection.find(q);

			return out.size();

		} else if (fromDate != null && untilDate != null) {

			BasicDBObject query = new BasicDBObject();

			query.put("$gt", fromDate.doubleValue());

			query.put("$lt", untilDate.doubleValue());

			return collection.getFileList(new BasicDBObject("metadata.timestamp", query)).size();

		} else if (tags != null) {

			QueryBuilder qBuilder = QueryBuilder.start("metadata.tags").in(tags);

			DBObject q = qBuilder.get();

			List<GridFSDBFile> out = collection.find(q);

			return out.size();

		}

		return 0;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)

	 */

	@Override

	public void setResultSet(final ResultSet resultSet) {

		resultSet.close();

	}

	/**

	 * Gets the from date.

	 *

	 * @return the from date

	 */

	public Long getFromDate() {

		return fromDate;

	}

	/**

	 * Sets the from date.

	 *

	 * @param fromdate

	 *            the new from date

	 */

	public void setFromDate(final Long fromdate) {

		this.fromDate = fromdate;

	}

	/**

	 * Gets the until date.

	 *

	 * @return the until date

	 */

	public Long getUntilDate() {

		return untilDate;

	}

	/**

	 * Sets the until date.

	 *

	 * @param untilDate

	 *            the new until date

	 */

	public void setUntilDate(final Long untilDate) {

		this.untilDate = untilDate;

	}

	/**

	 * Gets the records.

	 *

	 * @return the records

	 */

	public List<String> getRecords() {

		return records;

	}

	/**

	 * Sets the records.

	 *

	 * @param records

	 *            the new records

	 */

	public void setRecords(final List<String> records) {

		this.records = records;

	}

	/**

	 * Gets the collection.

	 *

	 * @return the collection

	 */

	public GridFS getCollection() {

		return collection;

	}

	/**

	 * Sets the collection.

	 *

	 * @param collection

	 *            the new collection

	 */

	public void setCollection(final GridFS collection) {

		this.collection = collection;

	}

	/**

	 * Gets the base uri.

	 *

	 * @return the base uri

	 */

	public String getBaseURI() {

		return baseURI;

	}

	/**

	 * Sets the base uri.

	 *

	 * @param baseURI

	 *            the new base uri

	 */

	public void setBaseURI(final String baseURI) {

		this.baseURI = baseURI;

	}

	/**

	 * Gets the object store id.

	 *

	 * @return the object store id

	 */

	public String getObjectStoreID() {

		return objectStoreID;

	}

	/**

	 * Sets the object store id.

	 *

	 * @param objectStoreID

	 *            the new object store id

	 */

	public void setObjectStoreID(final String objectStoreID) {

		this.objectStoreID = objectStoreID;

	}

	/**

	 * @return the tags

	 */

	public Set<String> getTags() {

		return tags;

	}

	/**

	 * @param tags

	 *            the tags to set

	 */

	public void setTags(final Set<String> tags) {

		this.tags = tags;

	}

}

package eu.dnetlib.data.objectstore.modular.gridFS;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

import java.util.Set;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.collect.Sets;

import com.mongodb.BasicDBList;

import com.mongodb.DBObject;

import com.mongodb.gridfs.GridFSDBFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * The Class ObjectStoreFileBuilder generates an objectStoreFile bean

 *

 */

public class ObjectStoreFileUtility {

	private static final int KB_SIZE = 1024;

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);

	/**

	 * Builds the.

	 *

	 * @param input

	 *            the input File

	 * @param baseURI

	 *            the base URI to associate to the metadata

	 * @param objectStoreID

	 *            The objectStoreID the input file

	 * @return the object store file

	 */

	public static ObjectStoreFile build(final GridFSDBFile input, final String baseURI, final String objectStoreID) {

		DBObject metadata = input.getMetaData();

		String originalFile = (String) metadata.get("originalObject");

		ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);

		ObjectStoreFile newFile = new ObjectStoreFile();

		newFile.setObjectID((String) metadata.get("id"));

		newFile.setAccessProtocol(Protocols.HTTP);

		newFile.setMimeType((String) metadata.get("mime"));

		newFile.setMd5Sum((String) input.get("md5"));

		newFile.setFileSizeKB(input.getLength() / KB_SIZE);

		BasicDBList metadataTag = (BasicDBList) metadata.get("tags");

		if (metadataTag != null) {

			Set<String> tagFiles = Sets.newHashSet();

			for (Object tag : metadataTag) {

				tagFiles.add((String) tag);

			}

			newFile.setTags(tagFiles);

		}

		if (originalFile != null) {

			newFile.setMetadataRelatedID(original.getMetadataRelatedID());

			if (original.getDownloadedURL() == null || original.getDownloadedURL().length() == 0) {

				newFile.setDownloadedURL(original.getURI());

			} else {

				newFile.setDownloadedURL(original.getDownloadedURL());

			}

		}

		try {

			newFile.setURI(baseURI + "?objectStore=" + URLEncoder.encode(objectStoreID, "UTF-8") + "&objectId="

					+ URLEncoder.encode(newFile.getObjectID(), "UTF-8"));

		} catch (UnsupportedEncodingException e) {

			log.error("Error on Build objectStoreFile ", e);

		}

		return newFile;

	}

	public static UnaryFunction<String, GridFSDBFile> asJSON(final String baseURI, final String objectStoreID) {

		return new UnaryFunction<String, GridFSDBFile>() {

			@Override

			public String evaluate(final GridFSDBFile input) {

				return build(input, baseURI, objectStoreID).toJSON();

			}

		};

	}

}

package eu.dnetlib.data.objectstore.modular.gridFS;

import java.util.List;

import java.util.regex.Pattern;

import org.bson.BSONObject;

import org.springframework.beans.factory.annotation.Required;

import com.mongodb.BasicDBObject;

import com.mongodb.DB;

import com.mongodb.DBCollection;

import com.mongodb.DBObject;

import com.mongodb.WriteConcern;

import com.mongodb.gridfs.GridFS;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;

import eu.dnetlib.miscutils.collections.MappedCollection;

import eu.dnetlib.miscutils.functional.UnaryFunction;

@SuppressWarnings("all")

public class GridFSObjectstoreDaoImpl implements ObjectStoreDao {

	public static final String INTERPRETATION = "interpretation";

	private final static String OBJECTSTOREMETADATANAME = "metadataObjectStore";

	private final static String OBJECTSTOREIDFIELD = "obsId";

	private DB db;

	private boolean upsert;

	private String objectStoreRESTURI;

	@Override

	public ObjectStore getObjectStore(String obsId) {

		db.setWriteConcern(WriteConcern.SAFE);

		obsId = obsId.substring(0, 36);

		GridFSObjectStore objectStore = new GridFSObjectStore(obsId, new GridFS(db, obsId), this.upsert);

		objectStore.setBaseURI(objectStoreRESTURI);

		checkIndexes(obsId);

		return objectStore;

	}

	private void checkIndexes(final String id) {

		DBCollection coll = db.getCollection(id + ".files");

		List<DBObject> indexInfo = coll.getIndexInfo();

		boolean hasTimestampIndex = false;

		for (DBObject info : indexInfo) {

			BSONObject boj = (BSONObject) info.get("key");

			if (boj.get("metadata.timestamp") != null) {

				hasTimestampIndex = true;

			}

		}

		if (!hasTimestampIndex) {

			coll.createIndex(new BasicDBObject("metadata.timestamp", 1));

		}

	}

	@Override

	public List<String> listObjectStores() {

		DBCollection metadata = db.getCollection(OBJECTSTOREMETADATANAME);

		return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() {

			@Override

			public String evaluate(final DBObject object) {

				return (String) object.get(OBJECTSTOREIDFIELD);

			}

		});

	}

	@Override

	public boolean createObjectStore(final String obsId, final String interpretation) {

		DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME);

		final BasicDBObject obj = new BasicDBObject();

		obj.put(OBJECTSTOREIDFIELD, obsId);

		obj.put(INTERPRETATION, interpretation);

		coll.save(obj);

		return true;

	}

	@Override

	public boolean updateObjectStore(final String obsId, final String interpretation) {

		DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME);

		final BasicDBObject obj = new BasicDBObject();

		obj.put(OBJECTSTOREIDFIELD, obsId);

		obj.put(INTERPRETATION, interpretation);

		coll.update(new BasicDBObject(OBJECTSTOREIDFIELD, obsId), obj, true, false);

		coll.save(obj);

		return true;

	}

	@Override

	public boolean deleteObjectStore(String obsId) {

		DBCollection coll = db.getCollection(OBJECTSTOREMETADATANAME);

		coll.remove(new BasicDBObject(OBJECTSTOREIDFIELD, obsId));

		obsId = obsId.substring(0, 36);

		GridFS objectStore = new GridFS(db, obsId);

		Pattern any = Pattern.compile(".");

		BasicDBObject query = new BasicDBObject("md5", any);

		objectStore.remove(query);

		return true;

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public boolean isUpsert() {

		return upsert;

	}

	@Required

	public void setUpsert(final boolean upsert) {

		this.upsert = upsert;

	}

	public String getObjectStoreRESTURI() {

		return objectStoreRESTURI;

	}

	@Required

	public void setObjectStoreRESTURI(final String objectStoreRESTURI) {

		this.objectStoreRESTURI = objectStoreRESTURI;

	}

}

services.objectStore.mongodb.host=localhost

services.objectStore.mongodb.port=27017

services.objectStore.mongodb.db=objectStore

services.objectStore.mongodb.connectionsPerHost=20

services.objectStore.mongodb.upsert=true

services.objectStore.RESTURI=http://${container.hostname}:${container.port}/${container.context}/mvc/objectStore/retrieve.do

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="gridFSObjectstoreDao"

	 		class="eu.dnetlib.data.objectstore.modular.gridFS.GridFSObjectstoreDaoImpl"

	 		p:db-ref="objectstoreMongoDB"

	 		p:objectStoreRESTURI-ref="restURIString"

	 		p:upsert="${services.objectStore.mongodb.upsert}"	 		

	 		/>

	  <bean id="restURIString" class="java.lang.String">

    		<constructor-arg value="${services.objectStore.RESTURI}"/>

    </bean>  

	 <bean id="objectStoreMongoServer" class="com.mongodb.Mongo">

		<constructor-arg index="0" type="com.mongodb.ServerAddress">

			<bean class="com.mongodb.ServerAddress">

				<constructor-arg index="0"

					value="${services.objectStore.mongodb.host}" />

				<constructor-arg index="1"

					value="${services.objectStore.mongodb.port}" />

			</bean>

		</constructor-arg>

		<constructor-arg index="1" type="com.mongodb.MongoOptions">

			<bean class="eu.dnetlib.data.objectstore.modular.gridFS.MongoGridFSOptionsFactory"

				p:connectionsPerHost="${services.objectStore.mongodb.connectionsPerHost}"				

				 />

		</constructor-arg>		

	</bean>

	<bean id="objectstoreMongoDB" factory-bean="objectStoreMongoServer"

		factory-method="getDB">

		<constructor-arg index="0"

			value="${services.objectStore.mongodb.db}" />

	</bean>

	</beans>