D-Net

package eu.dnetlib.data.collector.plugins.filesystem;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.Iterator;

import java.util.Set;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.collect.Iterators;

import com.google.common.collect.Sets;

/**

 * Class enabling lazy & recursive iteration of a filesystem tree. The iterator iterates over file paths.

 *

 * @author Andrea

 *

 */

public class FileSystemIterator implements Iterator<String> {

	/** The logger */

	private static final Log log = LogFactory.getLog(FileSystemIterator.class);

	private Set<String> extensions = Sets.newHashSet();

	private Iterator<Path> pathIterator;

	private String current;

	public FileSystemIterator(final String baseDir, final String extensions) {

		if(StringUtils.isNotBlank(extensions)) {

			this.extensions = Sets.newHashSet(extensions.split(","));

		}

		try {

			this.pathIterator = Files.newDirectoryStream(Paths.get(baseDir)).iterator();

			this.current = walkTillNext();

		} catch (IOException e) {

			log.error("Cannot initialize File System Iterator. Is this path correct? " + baseDir);

			throw new RuntimeException("Filesystem collection error.", e);

		}

	}

	@Override

	public boolean hasNext() {

		return current != null;

	}

	@Override

	public synchronized String next() {

		String pivot = new String(current);

		current = walkTillNext();

		log.debug("Returning: " + pivot);

		return pivot;

	}

	@Override

	public void remove() {}

	/**

	 * Walk the filesystem recursively until it finds a candidate. Strategies: a) For any directory found during the walk, an iterator is

	 * built and concat to the main one; b) Any file is checked against admitted extensions

	 *

	 * @return the next element to be returned by next call of this.next()

	 */

	private synchronized String walkTillNext() {

		while (pathIterator.hasNext()) {

			Path nextFilePath = pathIterator.next();

			if (Files.isDirectory(nextFilePath)) {

				// concat

				try {

					pathIterator = Iterators.concat(pathIterator, Files.newDirectoryStream(nextFilePath).iterator());

					log.debug("Adding folder iterator: " + nextFilePath.toString());

				} catch (IOException e) {

					log.error("Cannot create folder iterator! Is this path correct? " + nextFilePath.toString());

					return null;

				}

			} else {

				if (extensions.isEmpty() || extensions.contains(FilenameUtils.getExtension(nextFilePath.toString()))) {

					log.debug("Returning: " + nextFilePath.toString());

					return nextFilePath.toString();

				}

			}

		}

		return null;

	}

}