D-Net

package eu.dnetlib.msro.workers.aggregation.collect.plugins.archive.zip;

import java.io.File;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.stream.Stream;

import eu.dnetlib.miscutils.streams.DnetStreamSupport;

import eu.dnetlib.msro.workers.aggregation.collect.CollectException;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;

import eu.dnetlib.msro.workers.aggregation.collect.plugins.oai.engine.XmlCleaner;

import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;

import org.springframework.stereotype.Component;

// import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;

/**

 * Collector pluging for collecting a zipped folder of records

 *

 * @author Andrea

 */

@Component

@DnetCollectorPlugin("zip")

public class ZipCollectorPlugin implements CollectorPlugin {

	@Override

	public Stream<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		if ((baseUrl == null) || baseUrl.isEmpty()) { throw new CollectException("Param 'baseurl' is null or empty"); }

		try {

			final String zipPath = interfaceDescriptor.getBaseUrl();

			final URL zipUrl = new URL(zipPath);

			final File zipFile = new File(zipUrl.getPath());

			if (!zipFile.exists()) { throw new CollectException(String.format("The base ULR %s, does not exist", zipFile.getPath())); }

			return DnetStreamSupport.stream(new ZipIterator(zipFile.getAbsolutePath()))

					.map(s -> s.startsWith("\uFEFF") ? s.substring(1) : s)

					.map(XmlCleaner::cleanAllEntities);

		} catch (final MalformedURLException e) {

			throw new CollectException("Zip collector failed! ", e);

		}

	}

}