D-Net

package eu.dnetlib.data.mdstore.plugins;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.util.Arrays;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.commons.net.ftp.FTPClient;

import org.apache.commons.net.ftp.FTPFile;

import org.apache.commons.net.ftp.FTPSClient;

import com.google.common.net.UrlEscapers;

import eu.dnetlib.data.mdstore.plugins.objects.MdRecord;

import eu.dnetlib.data.mdstore.plugins.objects.MyURL;

public class EnrichLocalLinksPlugin extends MdRecordPlugin {

	private static final String DEFAULT_RIGHTS = "Open Access";

	private static final String INFO_FILENAME = "info.txt";

	private static final Log log = LogFactory.getLog(EnrichLocalLinksPlugin.class);

	private FTPClient ftpClient;

	private String hostedBy;

	private String baseUrl;

	private String ftpServer;

	private String ftpUser;

	private String ftpPassword;

	private String ftpBaseDir;

	private boolean ftpSecure = false;

	@Override

	protected void reconfigure(final Map<String, String> params) {

		setHostedBy(params.get("hostedBy"));

		setBaseUrl(params.get("baseUrl"));

		setFtpServer(params.get("ftpServer"));

		setFtpUser(params.get("ftpUser"));

		setFtpPassword(params.get("ftpPassword"));

		setFtpBaseDir(params.get("ftpBaseDir"));

		setFtpSecure(params.containsKey("ftpSecure") && params.get("ftpSecure").equalsIgnoreCase("true"));

		if (!getFtpBaseDir().startsWith("/")) {

			setFtpBaseDir("/" + getFtpBaseDir());

		}

		try {

			if (isFtpSecure()) {

				ftpClient = new FTPSClient();

				ftpClient.connect(getFtpServer());

				// Set protection buffer size

				((FTPSClient) ftpClient).execPBSZ(0);

				// Set data channel protection to private

				((FTPSClient) ftpClient).execPROT("P");

			} else {

				ftpClient = new FTPClient();

				ftpClient.connect(getFtpServer());

			}

			if (!ftpClient.login(getFtpUser(), getFtpPassword())) { throw new RuntimeException("FTP login failed"); }

			ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);

			ftpClient.enterLocalPassiveMode();

			ftpClient.setBufferSize(1024);

			log.info("Connected to " + ftpServer);

		} catch (final IOException e) {

			log.error("Connection Failed");

			throw new RuntimeException(e);

		}

	}

	@Override

	protected void resetConfiguration() {

		if (ftpClient.isConnected()) {

			try {

				ftpClient.disconnect();

				log.info("Disconnected from " + ftpServer);

			} catch (final IOException e) {

				log.error("Disconnection Failed");

				throw new RuntimeException(e);

			}

		}

		setHostedBy(null);

		setBaseUrl(null);

		setFtpServer(null);

		setFtpUser(null);

		setFtpPassword(null);

		setFtpBaseDir(null);

		setFtpClient(null);

		setFtpSecure(false);

	}

	@Override

	protected boolean updateRecord(final String recordId, final MdRecord doc) {

		final int year = doc.getDate();

		final String code = StringUtils.substringAfterLast(doc.getId(), ":");

		log.info(String.format("Processing record: %s (%s/%s)", doc.getId(), year, code));

		final List<String> files = touchAndListDir(doc.getTitle(), doc.getCreators(), doc.getType(), year, code);

		if (files.isEmpty()) {

			return false;

		} else {

			log.info("  - adding new urls: " + files.size());

			doc.setBestRights(DEFAULT_RIGHTS);

			for (final String f : files) {

				doc.getUrls().add(new MyURL(calculateUrl(recordId, code, year, f), getHostedBy(), DEFAULT_RIGHTS));

			}

		}

		return true;

	}

	private List<String> touchAndListDir(final String title, final Set<String> authors, final String type, final int year, final String code) {

		final String content = String.format(

				"TITLE     : %s\nAUTHOR(S) : %s\nTYPE      : %s\nYEAR      : %s\nCODE      : %s\n\n*** DO NOT EDIT THIS FILE ***\n\n",

				title,

				StringUtils.join(authors, ", "),

				type,

				year,

				code);

		if (ftpChangeDir(getFtpBaseDir()) && ftpChangeDir(Integer.toString(year)) && ftpChangeDir(code)) {

			try (InputStream is = new ByteArrayInputStream(content.getBytes())) {

				if (log.isDebugEnabled()) {

					log.debug(String.format(" - Saving file %s/%s/%s/%s", getFtpBaseDir(), year, code, INFO_FILENAME));

					log.debug(content);

				}

				if (!ftpClient.storeFile(INFO_FILENAME, is)) {

					log.error("Error saving file: " + ftpClient.getReplyCode() + " - " + ftpClient.getReplyString());

					throw new RuntimeException("Error saving file: " + ftpClient.getReplyString());

				}

			} catch (final IOException e) {

				log.error("Error saving info file");

				throw new RuntimeException("Error saving info file", e);

			}

			try {

				return Arrays.stream(ftpClient.listFiles())

						.map(FTPFile::getName)

						.filter(s -> s.toLowerCase().endsWith(".pdf"))

						.sorted()

						.collect(Collectors.toList());

			} catch (final IOException e) {

				log.error("Error listing files");

				throw new RuntimeException("Error listing files", e);

			}

		} else {

			log.error(String.format("Directory not found: %s/%s/%s", getFtpBaseDir(), year, code));

			throw new RuntimeException(String.format("Directory not found: %s/%s/%s", getFtpBaseDir(), year, code));

		}

	}

	private boolean ftpChangeDir(final String dir) {

		try {

			if (!ftpClient.changeWorkingDirectory(dir)) {

				ftpClient.makeDirectory(dir);

				return ftpClient.changeWorkingDirectory(dir);

			}

			return true;

		} catch (final IOException e) {

			log.error("Error changing or create dir: " + dir);

			throw new RuntimeException("Error changing or create dir: " + dir, e);

		}

	}

	private String calculateUrl(final String id, final String code, final int year, final String f) {

		// the parameter ID is necessary for a better integration with OpenAIRE

		return String.format("%s/%s/%s/%s?id=%s", getBaseUrl(), year, code, UrlEscapers.urlPathSegmentEscaper().escape(f),

				UrlEscapers.urlFormParameterEscaper().escape(id));

	}

	public String getHostedBy() {

		return hostedBy;

	}

	public void setHostedBy(final String hostedBy) {

		this.hostedBy = hostedBy;

	}

	public String getBaseUrl() {

		return baseUrl;

	}

	public void setBaseUrl(final String baseUrl) {

		this.baseUrl = baseUrl;

	}

	public String getFtpBaseDir() {

		return ftpBaseDir;

	}

	public void setFtpBaseDir(final String ftpBaseDir) {

		this.ftpBaseDir = ftpBaseDir;

	}

	public FTPClient getFtpClient() {

		return ftpClient;

	}

	public void setFtpClient(final FTPClient ftpClient) {

		this.ftpClient = ftpClient;

	}

	public String getFtpServer() {

		return ftpServer;

	}

	public void setFtpServer(final String ftpServer) {

		this.ftpServer = ftpServer;

	}

	public String getFtpUser() {

		return ftpUser;

	}

	public void setFtpUser(final String ftpUser) {

		this.ftpUser = ftpUser;

	}

	public String getFtpPassword() {

		return ftpPassword;

	}

	public void setFtpPassword(final String ftpPassword) {

		this.ftpPassword = ftpPassword;

	}

	public boolean isFtpSecure() {

		return ftpSecure;

	}

	public void setFtpSecure(final boolean ftpSecure) {

		this.ftpSecure = ftpSecure;

	}

}