D-Net

package eu.dnetlib.msro.workers.aggregation.collect.plugins.filesystem;

import java.io.File;

import java.io.FileInputStream;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.stream.Stream;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.stereotype.Component;

import eu.dnetlib.miscutils.streams.DnetStreamSupport;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorParam;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.ProtocolParameterType;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.oai.engine.XmlCleaner;

import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;

import eu.dnetlib.msro.workflows.nodes.collect.CollectException;

/**

 * @author andrea

 */

@Component

@DnetCollectorPlugin(value = "filesystem", parameters = {

		@DnetCollectorParam(value = "extensions", type = ProtocolParameterType.LIST)

})

public class FilesystemCollectorPlugin implements CollectorPlugin {

	private static final Log log = LogFactory.getLog(FilesystemCollectorPlugin.class);

	@Override

	public Stream<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		if ((baseUrl == null) || baseUrl.isEmpty()) { throw new CollectException("Param 'baseurl' is null or empty"); }

		try {

			final URL basePath = new URL(baseUrl);

			final File baseDir = new File(basePath.getPath());

			if (!baseDir.exists()) { throw new CollectException(String.format("The base ULR %s, does not exist", basePath.getPath())); }

			final String extension = interfaceDescriptor.getParams().get("extensions");

			return DnetStreamSupport.stream(new FileSystemIterator(baseDir.getAbsolutePath(), extension))

					.map(inputFileName -> {

						try (FileInputStream fileInputStream = new FileInputStream(inputFileName)) {

							final String s = IOUtils.toString(fileInputStream);

							return XmlCleaner.cleanAllEntities(s.startsWith("\uFEFF") ? s.substring(1) : s);

						} catch (final Exception e) {

							log.error("Unable to read " + inputFileName);

							return "";

						}

					});

		} catch (final MalformedURLException e) {

			throw new CollectException("Filesystem collector failed! ", e);

		}

	}

}