D-Net

package eu.dnetlib.data.collector.plugins.filesystem;

import java.io.FileInputStream;

import java.io.IOException;

import java.nio.file.FileVisitResult;

import java.nio.file.FileVisitor;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.nio.file.SimpleFileVisitor;

import java.nio.file.attribute.BasicFileAttributes;

import java.util.HashMap;

import java.util.Iterator;

import org.apache.commons.io.IOUtils;

import org.junit.Assert;

import org.junit.Ignore;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.filesystem.FilesystemCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

@Ignore

public class FileSystemCollectorPluginTest {

	Path edh = Paths.get("/var/lib/eagle/content/EDH");

	Path dai = Paths.get("/var/lib/eagle/content/DAI/arachne-eagle-images-v1-flat");

	Path datacite = Paths.get("/media/andrea/xfs/datacite/output");

	@Ignore

	@Test

	public void testCollection() throws CollectorServiceException {

		InterfaceDescriptor descr = new InterfaceDescriptor();

		HashMap<String, String> params = new HashMap<String, String>();

		params.put("extensions", "xml");

		descr.setBaseUrl("file:///var/lib/eagle/content/EDH");

		descr.setParams(params);

		FilesystemCollectorPlugin plugin = new FilesystemCollectorPlugin();

		Iterable<String> result = plugin.collect(descr, null, null);

		int counter = 0;

		double totalTime = 0;

		long lastTimestamp = System.currentTimeMillis();

		for (String s : result) {

			counter++;

			if (counter % 10000 == 0) {

				double deltaT = (System.currentTimeMillis() - lastTimestamp) / 1000.00;

				totalTime += deltaT;

				System.out.println("10K records collected in " + deltaT + " seconds");

				lastTimestamp = System.currentTimeMillis();

			}

			Assert.assertNotNull(s);

		}

		System.out.println("Total " + counter + " in " + totalTime + " seconds");

	}

	@Ignore

	@Test

	public void testJavaNioDirectoryStream() throws IOException {

		int counter = 0;

		double totalTime = 0;

		long lastTimestamp = System.currentTimeMillis();

		Iterator<Path> pathIterator = Files.newDirectoryStream(edh).iterator();

		while (pathIterator.hasNext()) {

			Path next = pathIterator.next();

			FileInputStream fileInputStream = new FileInputStream(next.toString());

			String s = IOUtils.toString(fileInputStream);

			counter++;

			if (counter % 10000 == 0) {

				double deltaT = (System.currentTimeMillis() - lastTimestamp) / 1000.00;

				totalTime += deltaT;

				System.out.println("10K records collected in " + deltaT + " seconds");

				lastTimestamp = System.currentTimeMillis();

			}

			Assert.assertNotNull(s);

			fileInputStream.close();

		}

		System.out.println("Total " + counter + " in " + totalTime + " seconds");

	}

	@Test

	public void testJavaNioWalkTree() throws IOException {

		FileVisitor<Path> fv = new SimpleFileVisitor<Path>() {

			int counter = 0;

			double totalTime = 0;

			long lastTimestamp = System.currentTimeMillis();

			@Override

			public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {

				FileInputStream fileInputStream = new FileInputStream(file.toString());

				String s = IOUtils.toString(fileInputStream);

				Assert.assertNotNull(s);

				counter++;

				if (counter % 10000 == 0) {

					double deltaT = (System.currentTimeMillis() - lastTimestamp) / 1000.00;

					totalTime += deltaT;

					System.out.println("10K records collected in " + deltaT + " seconds");

					lastTimestamp = System.currentTimeMillis();

				}

				fileInputStream.close();

				return FileVisitResult.CONTINUE;

			}

		};

		try {

			Files.walkFileTree(edh, fv);

		} catch (IOException e) {

			e.printStackTrace();

		}

	}

}