D-Net

package eu.dnetlib.data.objectstore.filesystem;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.regex.Pattern;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.model.Filters;

import com.mongodb.client.result.DeleteResult;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.collections.Pair;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * The Class FileSystemObjectStore.

 *

 * @author sandro

 */

	/**

	 *

	 */

	private static final String URI_FIELD = "uri";

	/**

	 *

	 */

	private static final String FS_PATH_FIELD = "fsPath";

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(FileSystemObjectStore.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The id. */

	private final String id;

	/** The interpretation. */

	private final String interpretation;

	/** The base path. */

	private final String basePath;

	/** The base uri. */

	/** The mongo metadata. */

	/**

	 * Instantiates a new file system object store.

	 *

	 *            the identifier

	 * @param interpretation

	 *            the interpretation

	 * @param basePath

	 *            the base path

	 * @param mongoMetadata

	 *            the mongo metadata

	 * @param baseURI

	 *            the base uri

		this.basePath = basePath;

		this.interpretation = interpretation;

		this.mongoMetadata = mongoMetadata;

		this.baseURI = baseURI;

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public String getId() {

		return this.id;

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public String getInterpretation() {

		return this.interpretation;

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) throws ObjectStoreServiceException {

		for (ObjectStoreRecord record : records) {

		return addedCounter;

	}

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)

	 *

	 * This method handles the case of web crawl files and other cases when the metadata in mdstores are also the objects to put into the objectstores.

	 */

	@Override

	public int  feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException {

		Iterable<ObjectStoreRecord> it = Iterables.transform(records, new Function<MetadataObjectRecord, ObjectStoreRecord>() {

			@Override

			public ObjectStoreRecord apply(final MetadataObjectRecord metadataObjectRecord) {

				ObjectStoreRecord r = new ObjectStoreRecord();

				r.setInputStream(new ByteArrayInputStream(metadataObjectRecord.getRecord().getBytes()));

				ObjectStoreFile fileMetadata = new ObjectStoreFile();

				fileMetadata.setObjectID(metadataObjectRecord.getId());

				fileMetadata.setMimeType(metadataObjectRecord.getMime());

				r.setFileMetadata(fileMetadata);

				return r;

			}

		});

		return feed(it, incremental);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)

	 */

	@Override

	public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {

		return feedObject(record);

	}

	private String feedObject(final ObjectStoreRecord record) {

		if (record != null) {

						Files.createDirectories(objResolvedPath.getParent());

						log.debug("Folder " + objResolvedPath.getParent() + " created");

						Integer size = 0;

							size = infos.getValue();

						}

								ModularObjectStoreRESTService.retrieveURL(getBaseURI(), getBasePath(), getId(), record.getFileMetadata().getObjectID());

							BasicDBObject metadata = new BasicDBObject();

							metadata.put("id", record.getFileMetadata().getObjectID());

							metadata.put("mime", record.getFileMetadata().getMimeType());

							metadata.put("originalObject", record.getFileMetadata().toJSON());

							metadata.put("timestamp", timestamp);

							metadata.put("md5Sum", md5Sum);

							metadata.put("size", size);

							metadata.put(URI_FIELD, url);

						log.error("Something bad happen on inserting Record", e);

					} finally {

								record.getInputStream().close();

							} catch (Exception e) {

								log.error("Error on close inputStream", e);

							}

						}

			}

		}

		return null;

	/**

	 * {@inheritDoc}

	 */

	@Override

	public ResultSetListener deliver(final Long from, final Long until) throws ObjectStoreServiceException {

		resultSet.setUntilDate(until);

	/**

	 * {@inheritDoc}

	 */

	@Override

	public ResultSetListener deliverIds(final Iterable<String> ids) throws ObjectStoreServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {

		DBObject resultQuery = mongoMetadata.find(query).first();

		checkAndGetFsPathField(resultQuery, objectId);

		if (resultQuery == null || !resultQuery.containsField(FS_PATH_FIELD))

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found or missing " + FS_PATH_FIELD + " field");

		String pathStr = (String) resultQuery.get(FS_PATH_FIELD);

		if (StringUtils.isBlank(pathStr))

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " with blank " + FS_PATH_FIELD);

		return pathStr;

	}

	 * {@inheritDoc}

	 */

	@Override

	public int getSize() throws ObjectStoreServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public void deleteObject(final String objectId) throws ObjectStoreServiceException {

		DBObject response = mongoMetadata.find(query).first();

		String pathStr = checkAndGetFsPathField(response, objectId);

		Path path = FileSystems.getDefault().getPath(pathStr);

			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found in the assigned path " + path);

			Files.delete(path);

		} catch (IOException e) {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public String getObject(final String recordId) throws ObjectStoreServiceException {

		DBObject response = mongoMetadata.find(query).first();

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean existIDStartsWith(final String startId) throws ObjectStoreServiceException {

	public boolean dropContent() throws ObjectStoreServiceException {

		}

			FileSystemUtility.deleteFolderRecursive(baseDirPath);

		} catch (IOException e) {

			throw new ObjectStoreServiceException("Error on dropping store ", e);

		}

		log.info("Deleted folder" + baseDirPath.toString());

		if (!Files.exists(baseDirPath)) {

			log.info("Recreating folder " + baseDirPath);

			try {

				Files.createDirectory(baseDirPath);

			} catch (IOException e) {

				throw new ObjectStoreServiceException("Error on dropping store ", e);

			}

		}

		log.info("Dropped content for object store " + id + ". " + deleteResult.getDeletedCount() + " object(s) deleted.");

	}

	public String toString() {

		return "FileSystemObjectStore{" +

				", interpretation='" + getInterpretation() + '\'' +

				", basePath='" + getBasePath() + '\'' +

				", baseURI='" + getBaseURI() + '\'' +

	}

	 * Gets the base uri.

	 *

	 * @return the baseURI

	 */

	public String getBaseURI() {

		return baseURI;

	}

		return basePath;

	}