D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-mongo-mdstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "cnr-mongo-mdstore"}

package eu.dnetlib.mongodb;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.FactoryBean;

import com.mongodb.MongoOptions;

public class MongoOptionsFactory implements FactoryBean {

	private int connectionsPerHost;

	@Override

	public Object getObject() throws BeansException {

		MongoOptions opts = new MongoOptions();

		opts.connectionsPerHost = connectionsPerHost;

		return opts;

	}

	@SuppressWarnings("rawtypes")

	@Override

	public Class getObjectType() {

		return MongoOptions.class;

	}

	@Override

	public boolean isSingleton() {

		return false;

	}

	public int getConnectionsPerHost() {

		return connectionsPerHost;

	}

	public void setConnectionsPerHost(int connectionsPerHost) {

		this.connectionsPerHost = connectionsPerHost;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Pattern;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.DBCollection;

import com.mongodb.DBCursor;

import com.mongodb.DBObject;

import com.mongodb.QueryBuilder;

import eu.dnetlib.enabling.resultset.ResultSet;

import eu.dnetlib.enabling.resultset.ResultSetAware;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.miscutils.maps.ConcurrentSizedMap;

public class MongoResultSetListener implements ResultSetListener,

		ResultSetAware {

	private static final Log log = LogFactory

			.getLog(MongoResultSetListener.class); // NOPMD by marko on 11/24/08

													// 5:02 PM

	private ConcurrentSizedMap<Integer, String> lastKeys = new ConcurrentSizedMap<Integer, String>();

	private final DBObject byId = new BasicDBObject("id", 1);

	private UnaryFunction<String, DBObject> serializer;

	private Pattern filter;

	private DBCollection collection;

	public MongoResultSetListener(DBCollection collection, Pattern filter,

			UnaryFunction<String, DBObject> serializer) {

		this.collection = collection;

		this.filter = filter;

		this.serializer = serializer;

	}

	@Override

	public List<String> getResult(int fromPosition, int toPosition) {

		// log.info("getting mdstore resultset " + fromPosition + ", " +

		// toPosition);

		ArrayList<DBObject> page = null;

		String lastKey = lastKeys.get(fromPosition);

		if (lastKey != null)

			page = continueFrom(lastKey, toPosition - fromPosition + 1);

		else

			page = fetchNew(fromPosition - 1, toPosition - fromPosition + 1);

		if (!page.isEmpty()) {

			DBObject last = page.get(page.size() - 1);

			lastKeys.put(toPosition + 1, (String) last.get("id"));

		}

		return listMap(page, serializer);

	}

	private ArrayList<DBObject> fetchNew(int from, int size) {

		// StopWatch stopWatch = new CommonsLogStopWatch(log);

		try {

			DBCursor cursor = null;

			if (filter != null) {

				DBObject query = QueryBuilder.start("body").regex(filter).get();

				cursor = collection.find(query).sort(byId);

			} else

				cursor = collection.find().sort(byId);

			return Lists.newArrayList((Iterable<DBObject>) cursor.skip(from)

					.limit(size));

		} finally {

			// stopWatch.stop("mongodb.list.scan");

		}

	}

	private ArrayList<DBObject> continueFrom(String lastKey, int size) {

		// StopWatch stopWatch = new CommonsLogStopWatch(log);

		try {

			if (log.isDebugEnabled())

				log.debug("trying to continue from previous key: " + lastKey);

			DBObject query = null;

			if (filter != null)

				query = QueryBuilder.start("id").greaterThan(lastKey)

						.and("body").regex(filter).get();

			else

				query = QueryBuilder.start("id").greaterThan(lastKey).get();

			final DBCursor cursor = collection.find(query).sort(byId);

			return Lists.newArrayList((Iterable<DBObject>) cursor.limit(size));

		} finally {

			// stopWatch.stop("mongodb.list.resume");

		}

	}

	@Override

	public int getSize() {

		if (filter != null) {

			DBObject query = QueryBuilder.start("body").regex(filter).get();

			return (int) collection.find(query).count();

		}

		return (int) collection.getCount();

	}

	@Override

	public void setResultSet(ResultSet resultSet) {

		resultSet.close();

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb.utils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;

import eu.dnetlib.enabling.tools.AbstractSchedulable;

public class EnsureIndexJob extends AbstractSchedulable {

	private static final Log log = LogFactory.getLog(EnsureIndexJob.class); // NOPMD by marko on 11/24/08 5:02 PM

	private MDStoreDao dao;

	@Override

	protected void doExecute() {

		log.info("performing scheduled mdstore index check");

		for (String mdId : getDao().listMDStores()) {

			try {

				log.info("ensureindex for mdStoreId:" + mdId);

				((MongoMDStore) getDao().getMDStore(mdId)).ensureIndices();

			} catch (Throwable e) {

				log.warn("unable to reindex mdstore: " + mdId, e);

			}

		}

	}

	@Required

	public void setDao(MDStoreDao dao) {

		this.dao = dao;

	}

	public MDStoreDao getDao() {

		return dao;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb.utils;

import java.io.StringReader;

import java.util.List;

import javax.xml.ws.Endpoint;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.mongodb.BasicDBObject;

import com.mongodb.DBCollection;

import com.mongodb.DBObject;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

import eu.dnetlib.soap.EndpointReferenceBuilder;

public class MetadataCheckJob {

	private static final Log log = LogFactory.getLog(MetadataCheckJob.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * lookup locator.

	 */

	private ServiceLocator<ISLookUpService> lookupLocator;

	/**

	 * {@link Endpoint}

	 */

	private Endpoint endpoint;

	/**

	 * {@link EndpointReferenceBuilder}

	 */

	private EndpointReferenceBuilder<Endpoint> eprBuilder;	

	/**

	 * MDStore dao.

	 */

	private MDStoreDao dao;

	private boolean runOnStart;

	public void runOnStart() {

		if(isRunOnStart()) {

			log.info("running mdStore metadata check on service start");

			repairMetadata();

		}

	}

	/**

	 * Job execution method.

	 */

	public void repairMetadata() {

		DBCollection metadata = ((MongoMDStore) getDao().getMDStore("metadata")).getCollection();

		if (metadata.findOne() != null) {

			log.debug("\n mdStore metadata doesn't need to be repaired");

			return;

		}

		try {

			List<String> mdStores = getLookupLocator().getService().quickSearchProfile(

				"//RESOURCE_PROFILE[" +

					".//RESOURCE_TYPE/@value='MDStoreDSResourceType' and " +

					".//RESOURCE_URI/@value='" + getServiceAddress() + "']");

			log.debug("\n repairing mdstore metadata");

			for(String MDStoreProfile : mdStores)

				metadata.save(getMdInfo(MDStoreProfile));

			log.debug("\n FINISHED repairing mdstore metadata");

		} catch(ISLookUpException e) {

			throw new RuntimeException(e);

		} catch(DocumentException e ) {

			throw new RuntimeException(e);

		}			

	}

	/**

	 * Helper method, gets an MDStore profile and returns a DBObject.

	 * 

	 * @param MDStoreProfile

	 * @return a DBObject representing the metadata informations

	 * @throws DocumentException

	 */

	private DBObject getMdInfo(final String MDStoreProfile) throws DocumentException {

		Document doc = new SAXReader().read(new StringReader(MDStoreProfile));

		DBObject dbo = new BasicDBObject();

		dbo.put("mdId", doc.valueOf("//RESOURCE_IDENTIFIER/@value"));

		dbo.put("format", doc.valueOf("//METADATA_FORMAT/text()"));

		dbo.put("layout", doc.valueOf("//METADATA_FORMAT_LAYOUT/text()"));

		dbo.put("interpretation", doc.valueOf("//METADATA_FORMAT_INTERPRETATION/text()"));

		return dbo;

	}

	private String getServiceAddress() {

		return getEprBuilder().getAddress(getEndpoint()) + "?wsdl";

	}	

	@Required

	public void setEprBuilder(EndpointReferenceBuilder<Endpoint> eprBuilder) {

		this.eprBuilder = eprBuilder;

	}

	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {

		return eprBuilder;

	}

	@Required

	public void setEndpoint(Endpoint endpoint) {

		this.endpoint = endpoint;

	}

	public Endpoint getEndpoint() {

		return endpoint;

	}	

	@Required

	public void setDao(MDStoreDao dao) {

		this.dao = dao;

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setLookupLocator(ServiceLocator<ISLookUpService> lookupLocator) {

		this.lookupLocator = lookupLocator;

	}

	public ServiceLocator<ISLookUpService> getLookupLocator() {

		return lookupLocator;

	}

	@Required

	public void setRunOnStart(boolean runOnStart) {

		this.runOnStart = runOnStart;

	}

	public boolean isRunOnStart() {

		return runOnStart;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb.utils;

import com.google.common.base.Function;

import com.mongodb.DBObject;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class MDStoreUtils {

	public static UnaryFunction<String, DBObject> mdId() {

		return new UnaryFunction<String, DBObject>() {

			@Override

			public String evaluate(DBObject arg) {

				return (String) arg.get("mdId");

			}

		};

	}

	public static UnaryFunction<Boolean, DBObject> dboFilter(final String format, final String layout, final String interpretation) {

		return new UnaryFunction<Boolean, DBObject>() {

			@Override

			public Boolean evaluate(DBObject dbo) {

				return dbo.get("format").toString().equals(format) && dbo.get("layout").toString().equals(layout)

						&& dbo.get("interpretation").toString().equals(interpretation);

			}

		};

	}

	public static Function<DBObject, String> body() {

		return new Function<DBObject, String>() {

			@Override

			public String apply(DBObject dbo) {

				return (String) dbo.get("body");

			}

		};

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.mongodb.BasicDBObject;

import com.mongodb.DB;

import com.mongodb.DBCollection;

import eu.dnetlib.data.mdstore.modular.RecordParser;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;

import eu.dnetlib.miscutils.collections.FilteredCollection;

import eu.dnetlib.miscutils.collections.MappedCollection;

/**

 * Factory bean for MongoMDStore instances.

 * 

 * @author marko

 * 

 */

public class MDStoreDaoImpl implements MDStoreDao {

	private static final Log log = LogFactory.getLog(MDStoreDaoImpl.class);

	private DB db;

	private RecordParser recordParser;

	private boolean upsert;

	private boolean discardRecords = true;

	@Override

	public void createMDStore(String mdId, String format, String interpretation, String layout) {

		String internalId = mdId;

		if (internalId.contains("_")) {

			internalId = StringUtils.substringBefore(mdId, "_");

		}

		db.createCollection(internalId, null);

		DBCollection coll = db.getCollection("metadata");

		final BasicDBObject obj = new BasicDBObject();

		obj.put("mdId", mdId);

		obj.put("format", format);

		obj.put("interpretation", interpretation);

		obj.put("layout", layout);

		coll.save(obj);

	}

	@Override

	public void deleteMDStore(String mdId) {

		DBCollection coll = db.getCollection("metadata");

		String internalId = mdId;

		if (internalId.contains("_")) {

			internalId = StringUtils.substringBefore(mdId, "_");

		}

		log.info("delete mdId: " + mdId);

		coll.remove(new BasicDBObject("mdId", mdId));

		final DBCollection mdColl = db.getCollection(internalId);

		if (mdColl != null) {

			mdColl.drop();

		}

	}

	@Override

	public MDStore getMDStore(String mdId) {

		String internalId = mdId;

		if (internalId.contains("_")) {

			internalId = StringUtils.substringBefore(mdId, "_");

		}

		return new MongoMDStore(mdId, db.getCollection(internalId), recordParser, isUpsert(), isDiscardRecords());

	}

	@Override

	public List<String> listMDStores() {

		return MappedCollection.listMap(db.getCollection("metadata").find(), MDStoreUtils.mdId());

	}

	@Override

	public List<String> listMDStores(String format, String layout, String interpretation) {

		return MappedCollection.listMap(

				FilteredCollection.listFilter(getDb().getCollection("metadata").find(), MDStoreUtils.dboFilter(format, layout, interpretation)),

				MDStoreUtils.mdId());

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(DB db) {

		this.db = db;

	}

	public RecordParser getRecordParser() {

		return recordParser;

	}

	@Required

	public void setRecordParser(RecordParser recordParser) {

		this.recordParser = recordParser;

	}

	@Required

	public void setUpsert(boolean upsert) {

		this.upsert = upsert;

	}

	public boolean isUpsert() {

		return upsert;

	}

	public boolean isDiscardRecords() {

		return discardRecords;

	}

	public void setDiscardRecords(boolean discardRecords) {

		this.discardRecords = discardRecords;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.BlockingQueue;

import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.BasicBSONObject;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Iterators;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.DBCollection;

import com.mongodb.DBCursor;

import com.mongodb.DBObject;

import com.mongodb.QueryBuilder;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.modular.RecordParser;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.MappedCollection;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class MongoMDStore implements MDStore {

	private static final class SerializeMongoRecord implements UnaryFunction<String, DBObject> {

		@Override

		public String evaluate(final DBObject arg) {

			return (String) arg.get("body");

		}

	}

	private static final class SerializeMongoRecordId implements UnaryFunction<String, DBObject> {

		@Override

		public String evaluate(final DBObject arg) {

			return (String) arg.get("id");

		}

	}

	private static final Log log = LogFactory.getLog(MongoMDStore.class);

	private String id;

	private DBCollection collection;

	private DBCollection discardedCollection;

	private RecordParser recordParser;

	private final boolean upsert;

	private final boolean discardRecords;

	private static List<String> requiredIndicies = Lists.newArrayList("{ \"id\" : 1}", "{ \"timestamp\" : 1}", "{ \"originalId\" : 1}");

	public MongoMDStore(final String id, final DBCollection collection, final RecordParser recordParser, final boolean upsert, final boolean discardRecords) {

		this.id = id;

		this.collection = collection;

		this.discardedCollection = collection.getDB().getCollection("discarded-" + StringUtils.substringBefore(id, "_"));

		this.recordParser = recordParser;

		this.upsert = upsert;

		this.discardRecords = discardRecords;

	}

	@Override

	public int feed(final Iterable<String> records, final boolean incremental) {

		ensureIndices();

		final BlockingQueue<Object> queue = new ArrayBlockingQueue<Object>(80);

		final Object sentinel = new Object();

		Thread background = new Thread(new Runnable() {

			@Override

			public void run() {

				while (true) {

					try {

						Object record = queue.take();

						if (record == sentinel) {

							break;

						}

						safeFeedRecord((String) record, incremental);

					} catch (InterruptedException e) {

						log.fatal("got exception in background thread", e);

						throw new IllegalStateException(e);

					}

				}

			}

		});

		background.start();

		try {

			log.info("feeding mdstore " + id);

			for (final String record : records) {

				queue.put(record);

			}

			queue.put(sentinel);

			// safeFeedRecord(record, incremental);

			log.info("finished feeding mdstore " + id);

			background.join();

		} catch (InterruptedException e) {

			throw new IllegalStateException(e);

		}

		// double check

		ensureIndices();

		collection.ensureIndex(new BasicDBObject("id", 1));

		return getSize();

	}

	public void ensureIndices() {

		collection.resetIndexCache();

		for (String key : Lists.newArrayList("id", "timestamp", "originalId")) {

			collection.ensureIndex(new BasicDBObject(key, 1));

		}

	}

	public boolean isIndexed() {

		return Lists.newArrayList(new MappedCollection<String, DBObject>(collection.getIndexInfo(), new UnaryFunction<String, DBObject>() {

			@Override

			public String evaluate(final DBObject dbo) {

				return new BasicBSONObject(dbo.toMap()).getString("key");

			}

		})).containsAll(requiredIndicies);

	}

	private void safeFeedRecord(final String record, final boolean incremental) {

		try {

			if (record.isEmpty()) return;

			feedRecord(record, incremental);

		} catch (final Throwable e) {

			if (discardRecords) {

				log.info("Got unhandled exception while parsing record", e);

				discardedCollection.insert(new BasicDBObject("body", record));

			}

		}

	}

	private void feedRecord(final String record, final boolean incremental) {

		final Map<String, String> recordProperties = recordParser.parseRecord(record);

		log.debug("found props: " + recordProperties);

		if (recordProperties.containsKey("id")) {

			DBObject obj = new BasicDBObject();

			final String id = recordProperties.get("id");

			final String originalId = recordProperties.get("originalId");

			obj.put("id", id);

			obj.put("originalId", originalId);

			obj.put("body", record);

			obj.put("timestamp", System.currentTimeMillis());

			collection.update(new BasicDBObject("id", id), obj, upsert, false);

		} else {

			if (discardRecords) {

				log.debug("parsed record seems invalid");

				discardedCollection.insert(new BasicDBObject("body", record));

			}

		}

	}

	/**

	 * Method searches for the given string grep into this collection and replaces it with the given replacement.

	 * 

	 * @param grep

	 *            the string to search

	 * @param replace

	 *            the replacement

	 */

	public void replace(final String grep, final String replace) {

		Pattern regex = Pattern.compile(grep, Pattern.MULTILINE);

		DBCursor matches = collection.find(QueryBuilder.start("body").regex(regex).get());

		log.debug("FOUND: " + matches.size());

		for (DBObject match : matches) {

			DBObject o = new BasicDBObject(match.toMap());

			o.put("body", regex.matcher((String) match.get("body")).replaceAll(replace));

			collection.update(match, o);

		}

	}

	@Override

	public ResultSetListener deliver(final String from, final String until, final String recordFilter) {

		return deliver(from, until, recordFilter, new SerializeMongoRecord());

	}

	@Override

	public ResultSetListener deliverIds(final String from, final String until, final String recordFilter) {

		return deliver(from, until, recordFilter, new SerializeMongoRecordId());

	}

	public ResultSetListener deliver(final String from, final String until, final String recordFilter, final UnaryFunction<String, DBObject> serializer) {

		ensureIndices();

		final Pattern filter = ((recordFilter != null) && (recordFilter.length() > 0)) ? Pattern.compile(recordFilter, Pattern.MULTILINE) : null;

		return new MongoResultSetListener(collection, filter, serializer);

	}

	@Override

	public Iterable<String> iterate() {

		return new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return Iterators.transform(collection.find(), MDStoreUtils.body());

			}

		};

	}

	@Override

	public void deleteRecord(final String recordId) {

		collection.remove(new BasicDBObject("id", recordId));

	}

	@Override

	public String getRecord(final String recordId) throws DocumentNotFoundException {

		DBObject obj = collection.findOne(new BasicDBObject("id", recordId));

		if ((obj == null) || (obj.containsField("body") == false)) throw new DocumentNotFoundException("The document with id " + id + " does not exist");

		String body = (String) obj.get("body");

		if (body.trim().length() == 0) throw new DocumentNotFoundException("The document with id " + id + " does not exist");

		return (new SerializeMongoRecord()).evaluate(obj);

	}

	@Override

	public void truncate() {

		collection.drop();

		discardedCollection.drop();

	}

	public DBObject getMDStoreMetadata() {

		return collection.getDB().getCollection("metadata").findOne(new BasicDBObject("mdId", getId()));

	}

	@Override

	public String getFormat() {

		return (String) getMDStoreMetadata().get("format");

	}

	@Override

	public String getInterpretation() {

		return (String) getMDStoreMetadata().get("interpretation");

	}

	@Override

	public String getLayout() {

		return (String) getMDStoreMetadata().get("layout");

	}

	@Override

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public DBCollection getCollection() {

		return collection;

	}

	public void setCollection(final DBCollection collection) {

		this.collection = collection;

	}

	public RecordParser getRecordParser() {

		return recordParser;

	}

	@Required

	public void setRecordParser(final RecordParser recordParser) {

		this.recordParser = recordParser;

	}

	@Override

	public int getSize() {

		return (int) collection.getCount();

	}

	public DBCollection getDiscardedCollection() {

		return discardedCollection;

	}

	public void setDiscardedCollection(final DBCollection discardedCollection) {

		this.discardedCollection = discardedCollection;

	}

}

package eu.dnetlib.data.mdstore.modular.inspector;

import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;

import java.util.List;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.stereotype.Controller;

import org.springframework.ui.Model;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.RequestParam;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mdstore.modular.MDStoreFeeder;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;

import eu.dnetlib.enabling.inspector.AbstractInspectorController;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.MappedCollection;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.miscutils.functional.string.EscapeHtml;

@Controller

public class MDStoreInspector extends AbstractInspectorController {

	private static final Log log = LogFactory.getLog(MDStoreInspector.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource(name="mongodbMDStoreDao")

	private MDStoreDao mdstoreDao;

	@Resource

	private MDStoreFeeder feeder;

	class MDStoreDescription {

		private String id;

		private int size;

		public MDStoreDescription(String id, int size) {

			super();

			this.id = id;

			this.size = size;

		}

		public String getFormat() {

			return mdstoreDao.getMDStore(id).getFormat();

		}

		public String getLayout() {

			return mdstoreDao.getMDStore(id).getLayout();

		}

		public String getInterpretation() {

			return mdstoreDao.getMDStore(id).getInterpretation();

		}

		public boolean getIndexed() {

			return ((MongoMDStore) mdstoreDao.getMDStore(id)).isIndexed();

		}

		public String getId() {

			return id;

		}

		public void setId(String id) {

			this.id = id;

		}

		public int getSize() {

			return size;

		}

		public void setSize(int size) {

			this.size = size;

		}

	}

	@RequestMapping(value = "/inspector/mdstores.do")

	public void mdstores(final Model model) {

		model.addAttribute("mdstores", MappedCollection.listMap(mdstoreDao.listMDStores(), new UnaryFunction<MDStoreDescription, String>() {

			@Override

			public MDStoreDescription evaluate(String arg) {

				MDStore mdstore = mdstoreDao.getMDStore(arg);

				return new MDStoreDescription(arg, mdstore.getSize());

			}

		}));

	}

	@RequestMapping(value = "/inspector/mdstore.do", method = RequestMethod.GET)

	public void mdstore(

			final Model model, 

			@RequestParam("id") String id, 

			@RequestParam(value = "start", required = false) Integer startParam,

			@RequestParam(value = "regex", required = false) String regex) {

		int pageSize = 10;

		int start = 0;

		if (startParam != null)

			start = startParam;

		MDStore mdstore = mdstoreDao.getMDStore(id);

		ResultSetListener rs = mdstore.deliver(null, null, regex);

		List<String> page = rs.getResult(1 + start, start + pageSize);

		model.addAttribute("id", id);

		model.addAttribute("start", start);

		model.addAttribute("regex", regex);

		model.addAttribute("nextPage", start + pageSize);

		model.addAttribute("prevPage", Math.max(0, start - pageSize));

		model.addAttribute("size", rs.getSize());

		model.addAttribute("page", listMap(page, new EscapeHtml()));

	}

	@RequestMapping(value = "/inspector/mdstore.do", method = RequestMethod.POST)

	public String bulkReplace(

			final Model model, 

			@RequestParam("id") String id,

			@RequestParam("regex") final String regex,

			@RequestParam("replace") final String replace,

			@RequestParam(value = "checkReplace", required = false) final Boolean checkReplace) {

		log.debug("regex: " + regex);

		log.debug("replace: " + replace);

		MongoMDStore mdstore = (MongoMDStore) mdstoreDao.getMDStore(id);

		boolean replaceEnable = checkReplace != null && checkReplace == true;

		if (replaceEnable)

			mdstore.replace(regex, replace);

		else

			model.addAttribute("regex", regex);

		return "redirect:mdstore.do?id=" + id;

	}	

	@RequestMapping(value = "/inspector/mdstoreEditResult.do")

	public void mdstoreEditRecord(final Model model, @RequestParam("id") String id, @RequestParam("start") int start, @RequestParam("index") int index) {

		MDStore mdstore = mdstoreDao.getMDStore(id);

		ResultSetListener rs = mdstore.deliver(null, null, null);

		List<String> page = rs.getResult(1 + index + start, 1 + index + start);

		model.addAttribute("id", id);

		model.addAttribute("start", start);

		model.addAttribute("record", new EscapeHtml().evaluate(Iterables.getOnlyElement(page)));

	}

	@RequestMapping(value = "/inspector/mdstoreSaveRecord.do")

	public String mdstoreSaveRecord(

			final Model model,

			@RequestParam("id") String id,

			@RequestParam("start") int start,

			@RequestParam("record") String record) {

		MDStore mdstore = mdstoreDao.getMDStore(id);

		mdstore.feed(Lists.newArrayList(record), true);

		return "redirect:mdstore.do?id=" + id + "&start=" + start;

	}

	@RequestMapping(value = "/inspector/mdstoreDeleteRecord.do")

	public String mdstoreDeleteRecord(final Model model, @RequestParam("id") String id, @RequestParam("start") int start, @RequestParam("index") int index) {

		MDStore mdstore = mdstoreDao.getMDStore(id);

		ResultSetListener rs = mdstore.deliverIds(null, null, null);

		List<String> page = rs.getResult(1 + index + start, 1 + index + start);

		String recordId = Iterables.getOnlyElement(page);

		log.fatal("deleting record " + recordId);

		mdstore.deleteRecord(recordId);

		return "redirect:mdstore.do?id=" + id + "&start=" + start;

	}

	@RequestMapping(value = "/inspector/mdstoresRefreshSizes.do")

	public String mdstoresRefreshSizes(final Model model) {

		for (String mdId : mdstoreDao.listMDStores())

			feeder.touchSize(mdId, mdstoreDao.getMDStore(mdId).getSize());

		return "redirect:mdstores.do";

	}

	@RequestMapping(value = "/inspector/ensure.do")

	public String mdstoreEnsureIndex(final Model model, @RequestParam("id") String id) {

		MongoMDStore mdStore = (MongoMDStore) mdstoreDao.getMDStore(id);

		log.info("manual ensureIndex for mdId: " + id);

		mdStore.ensureIndices();

		return "redirect:mdstores.do";

	}

}

$inspector/master(it={

<style type="text/css">

 .mdstoreRecords li.record {

   display: block;

   border-top: 1px solid #ccc;

 }

 .mdstoreRecords ul.nav li {

   display: inline;

 }

</style>

<h2>MDStore details</h2>

<p>

<form action="mdstore.do?id=$id$" method="POST">

GREP: <input name="regex" value="$regex$"/> REPLACEMENT: <input name="replace" value="$replace$"/>

Replace? <input type="checkbox" name="checkReplace" $if(checkReplace)$checked="checked"$endif$/>

<input type="submit" value="grep/repl"/>

</form>

</p>

<p>Show from: $start$ (<a href="?id=$id$&start=$prevPage$&regex=$regex$">prev</a>) out of $size$ (<a href="?id=$id$&start=$nextPage$&regex=$regex$">next</a>)</p>

<ul class="mdstoreRecords">

$page:{

  <li class="record">

   <ul class="nav">

     <li><a href="mdstoreEditResult.do?id=$id$&start=$start$&index=$i0$">edit</a></li>

     <li><a href="mdstoreDeleteRecord.do?id=$id$&start=$start$&index=$i0$">delete</a></li>

   </ul>

	 <pre>$it$</pre>

  </li>

}$

</ul>

})$

$inspector/master(it={

<h2>MDStores</h2>

<table>

  <tr>

    <th>Format</th>

    <th>Layout</th>

    <th>Interpretation</th>

    <th>Size</th>

    <th>ID</th>

    <th>Indexed?</th>

    <th>Ensure index</th>

  </tr>

  $mdstores:{

    <tr><td>$it.format$</td><td>$it.layout$</td><td>$it.interpretation$</td><td>$it.size$</td><td><a href="mdstore.do?id=$it.id$">$it.id$</a></td><td><span class="$if(it.indexed)$enabled$else$disabled$endif$">$it.indexed$</span></td><td><a href="ensure.do?id=$it.id$">GO</a></td></tr>

  }$

</table>

})$

$inspector/master(it={

<style type="text/css">

 #record {

  height: 60em;

  width: 100%;

  border: 1px solid #B0B0FF;

  padding: 4px;

 }

</style>

<h2>Low level edit mdstore record:</h2>

<ul>

  <li><a href="javascript:document.forms[0].submit()">save</a></li>

</ul>

<p> It will <b>not</b> trigger a mdstore modification date. </p>

<form action="mdstoreSaveRecord.do" method="POST">

<input type="hidden" name="id" value="$id$"/>

<input type="hidden" name="start" value="$start$"/>

<textarea id="record" name="record">

$record$

</textarea>

</form>

})$

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- beans -->

	<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean"

		depends-on="mdStoreService">

		<property name="triggers">

			<list>

				<ref bean="mdStoreCheckTrigger"/>

			</list>

		</property>

	</bean>

	<bean id="mdStoreEnsureIndexScheduler"

		class="eu.dnetlib.data.mdstore.modular.mongodb.utils.EnsureIndexJob"

		p:cronExpression="${services.mdstore.mongodb.ensureindex.cron}" 

		p:enabled="${services.mdstore.mongodb.ensureindex.enable}"

		p:dao-ref="mongodbMDStoreDao" />

	<bean id="mdStoreCheckTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean"

		p:startDelay="${services.mdstore.mongodb.checkmetadata.startdelay}" p:repeat-count="0" p:repeatInterval="60000">

		<property name="jobDetail">

			<bean class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">

				<property name="targetObject" ref="mdStoreCheckMetadataMBean" />

				<property name="targetMethod" value="runOnStart" />

				<property name="concurrent" value="false" />

			</bean>

		</property>

	</bean>		

	<bean id="mdStoreCheckMetadataMBean"

		class="eu.dnetlib.data.mdstore.modular.mongodb.utils.MetadataCheckJob" 

		depends-on="mdStoreService"		

		p:runOnStart="${services.mdstore.mongodb.checkmetadata.onstart}"

		p:dao-ref="mongodbMDStoreDao" p:lookupLocator-ref="lookupLocator" 

		p:endpoint-ref="mdStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />

	<bean id="jmxExporter" class="org.springframework.jmx.export.MBeanExporter">

		<property name="beans">

			<map>

				<entry key="mongodbMDStore:name=metadata" value-ref="mdStoreCheckMetadataMBean" />

			</map>

		</property>

	</bean>

</beans>

services.mdstore.mongodb.host=localhost

services.mdstore.mongodb.port=27017

services.mdstore.mongodb.db=mdstore

services.mdstore.mongodb.connectionsPerHost=20

services.mdstore.mongodb.ensureindex.cron=0 0 23 * * ?

services.mdstore.mongodb.ensureindex.enable=false

services.mdstore.mongodb.checkmetadata.onstart=true

services.mdstore.mongodb.checkmetadata.startdelay=30000

services.mdstore.mongodb.upsert=true

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="mongodbMDStoreDao" class="eu.dnetlib.data.mdstore.modular.mongodb.MDStoreDaoImpl"

		p:db-ref="mdstoreMongoDB" p:recordParser-ref="mdstoreRecordParser" 

		p:upsert="${services.mdstore.mongodb.upsert}"

		p:discardRecords="${services.mdstore.discardrecords}"/>

	<bean id="mdstoreMongoServer" class="com.mongodb.Mongo">

		<constructor-arg index="0" type="com.mongodb.ServerAddress">

			<bean class="com.mongodb.ServerAddress">

				<constructor-arg index="0"

					value="${services.mdstore.mongodb.host}" />

				<constructor-arg index="1"

					value="${services.mdstore.mongodb.port}" />

			</bean>

		</constructor-arg>

		<constructor-arg index="1" type="com.mongodb.MongoOptions">

			<bean class="eu.dnetlib.mongodb.MongoOptionsFactory"

				p:connectionsPerHost="${services.mdstore.mongodb.connectionsPerHost}" />

		</constructor-arg>

	</bean>

	<bean id="mdstoreMongoDB" factory-bean="mdstoreMongoServer"

		factory-method="getDB">

		<constructor-arg index="0"

			value="${services.mdstore.mongodb.db}" />

	</bean>

</beans>

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd

                http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">

	<bean id="mdstoreInspectorGroup"

		class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"

		p:name="mdstore">

		<property name="descriptors">

			<list>

				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"

					p:name="mdstores" p:relativeUrl="mdstores.do" />

			</list>

		</property>

	</bean>

</beans>

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0-SNAPSHOT</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>cnr-mongo-mdstore</artifactId>

	<packaging>jar</packaging>

	<version>1.1.0-SNAPSHOT</version>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-test</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.6</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-test-utils</artifactId>

			<version>[1.0.0-SNAPSHOT]</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-modular-mdstore-service</artifactId>

			<version>[2.0.0-SNAPSHOT]</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-inspector</artifactId>

			<version>[1.0.0-SNAPSHOT]</version>

		</dependency>

		<dependency>

			<groupId>org.mongodb</groupId>

			<artifactId>mongo-java-driver</artifactId>

			<version>${mongodb.driver.version}</version>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-mongo-mdstore/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "cnr-mongo-mdstore"}

package eu.dnetlib.data.mdstore.modular.mongodb;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import org.junit.After;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertEquals;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class MDStoreDaoImplTest {

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreTransactionManager manager;

	@After

	public void tearDown() throws MDStoreServiceException {

		dao.deleteMDStore("1");

		dao.deleteMDStore("2");

		dao.deleteMDStore("3");

		dao.deleteMDStore("4");

		dao.deleteMDStore("5");

		dao.deleteMDStore("6");

	}

	@Before

	public void setUp() throws Exception {

		dao.createMDStore("1", "F", "I", "L");

		dao.createMDStore("2", "F", "I", "L");

		dao.createMDStore("3", "F", "I", "L");

		dao.createMDStore("4", "F", "I", "L");

		dao.createMDStore("5", "F1", "I", "L");

		dao.createMDStore("6", "F1", "I", "L");

		final MongoCollection<DBObject> metadata = db.getCollection(MDStoreDaoImpl.METADATA_NAME, DBObject.class);

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "1"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "2"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "3"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "4"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "5"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "6"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

	}

	@Test

	public void test() throws MDStoreServiceException {

		assertEquals(40, dao.getSumOfSizes("F", "L", "I"));

		assertEquals(20, dao.getSumOfSizes("F1", "L", "I"));

		assertEquals(0, dao.getSumOfSizes("F_0", "L", "I"));

	}

	@Test

	public void getDBStatusTest() {

		final MDStoreDBStatus dbStatus = dao.getDBStatus();

		System.out.println(dbStatus);

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.util.Iterator;

import java.util.UUID;

import com.mongodb.DBObject;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.MDStoreServiceException;