D-Net

package eu.dnetlib.msro.workflows.nodes;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.index.ContextLoader;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.manager.MSROException;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

public class LoadContextsJobNode extends SimpleJobNode {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(LoadContextsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String contextParam;

	@Autowired

	private ContextLoader contextLoader;

	@Override

	protected String execute(final Env env) throws Exception {

		log.info("start preparing job");

		final String contexts = this.contextLoader.load();

		if (StringUtils.isBlank(contexts)) { throw new MSROException("empty contexts"); }

		env.setAttribute(getContextParam(), contexts);

		return Arc.DEFAULT_ARC;

	}

	public String getContextParam() {

		return contextParam;

	}

	public void setContextParam(final String contextParam) {

		this.contextParam = contextParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.provision;

import java.util.List;

import javax.annotation.Resource;

import com.google.common.collect.Iterables;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.enabling.ISLookUpService;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class FindIndexJobNode extends SimpleJobNode {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(FindIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource

	private UniqueServiceLocator serviceLocator;

	/**

	 * if non null, overrides format from env.

	 */

	private String mdFormat;

	/**

	 * if non null, overrides format from env.

	 */

	private String layout;

	/**

	 * if non null, overrides format from env.

	 */

	private String interpretation;

	/**

	 * {@inheritDoc}

	 */

	@Override

	public String execute(final Env env) {

		final String envFormat = env.getAttribute("format", String.class);

		final String envLayout = env.getAttribute("layout", String.class);

		final String envInterpretation = env.getAttribute("interpretation", String.class);

		final String format = handleOverride(env, "format", envFormat, getMdFormat());

		final String layout = handleOverride(env, "layout", envLayout, getLayout());

		final String interp = handleOverride(env, "interpretation", envInterpretation, getInterpretation());

		String mdRef = format + "-" + layout + "-" + interp;

		log.info("searching index for [" + mdRef + "]");

		final String indexId = findIndex(format, layout, interp);

		env.setAttribute("index_id", indexId);

		if (indexId == null || indexId.isEmpty()) {

			log.info("no index was found for [" + mdRef + "]");

			return "notFound";

		} else {

			log.info("index found for [" + mdRef + "]: " + indexId);

			return "found";

		}

	}

	private String findIndex(final String format, final String layout, final String interp) {

		final String xquery =

				"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexDSResourceType']" + "return $x[.//METADATA_FORMAT = '" + format

						+ "' and .//METADATA_FORMAT_LAYOUT='" + layout + "' and .//METADATA_FORMAT_INTERPRETATION = '" + interp

						+ "']//RESOURCE_IDENTIFIER/@value/string()";

		try {

			log.info("xquery: " + xquery);

			List<String> ids = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);

			log.info("found indexDS ids: " + ids);

			if (ids == null || ids.isEmpty()) { return null; }

			if (ids.size() > 1) {

				throw new IllegalStateException("found more than one index of given format: " + format + ", layout: " + layout

						+ ", interpretation: " + interp);

			}

			return Iterables.getOnlyElement(ids);

		} catch (Exception e) {

			return null;

		}

	}

	private String handleOverride(final Env env, final String name, final String envParam, final String override) {

		if (override != null) {

			env.setAttribute(name, override);

		}

		return override != null ? override : envParam;

	}

	public String getMdFormat() {

		return mdFormat;

	}

	public void setMdFormat(final String mdFormat) {

		this.mdFormat = mdFormat;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import java.util.List;

import javax.annotation.Resource;

import com.google.gson.Gson;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.enabling.ISLookUpException;

import eu.dnetlib.rmi.enabling.ISLookUpService;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class FetchRelClassesJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(FetchRelClassesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String xquery;

	private String relClassesName;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final Env env) throws Exception {

		final String relClasses = new Gson().toJson(fetchRelClasses());

		log.info("loaded relClasses: " + relClasses);

		env.setAttribute(getRelClassesName(), relClasses);

		return Arc.DEFAULT_ARC;

	}

	private List<String> fetchRelClasses() throws ISLookUpException {

		log.info("Loading relClasses with xquery: " + getXquery());

		return serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());

	}

	public String getRelClassesName() {

		return relClassesName;

	}

	public void setRelClassesName(final String relClassesName) {

		this.relClassesName = relClassesName;

	}

	public String getXquery() {

		return xquery;

	}

	public void setXquery(final String xquery) {

		this.xquery = xquery;

	}

}

package eu.dnetlib.msro.workflows.nodes.vre;

import java.net.URI;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.procs.Env;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.client.utils.URIBuilder;

import org.apache.http.entity.ContentType;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.BasicResponseHandler;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

/**

 * Created by alessia on 24/03/16.

 */

public class VREPostJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(VREPostJobNode.class);

	private String scheme;

	private String host;

	private String postPath;

	// POST parameters

	private String appToken;

	private String previewTitle;

	private String previewDescription;

	private String text;

	private String httpImageUrl;

	private boolean enableNotification;

	@Override

	protected String execute(final Env env) throws Exception {

		final CloseableHttpClient httpclient = HttpClients.createDefault();

		try {

			final URI postUri = new URIBuilder().setScheme(this.scheme).setHost(this.host).setPath(this.postPath).build();

			final HttpPost postAction = new HttpPost(postUri);

			final StringBuilder strBuilder = new StringBuilder();

			strBuilder.append("token=").append(this.appToken).append("&previewtitle=").append(this.previewTitle)

					.append("&previewdescription=").append(this.previewDescription).append("&text=").append(this.text)

					.append("&httpimageurl=").append(this.httpImageUrl).append("&enablenotification=").append(this.enableNotification);

			final String postBody = strBuilder.toString();

			log.debug("POST body: " + postBody);

			final StringEntity ent = new StringEntity(postBody, ContentType.APPLICATION_FORM_URLENCODED);

			postAction.setEntity(ent);

			final BasicResponseHandler responseHandler = new BasicResponseHandler();

			httpclient.execute(postAction, responseHandler);

			return Arc.DEFAULT_ARC;

		} finally {

			httpclient.close();

		}

	}

	public String getScheme() {

		return this.scheme;

	}

	public void setScheme(final String scheme) {

		this.scheme = scheme;

	}

	public String getHost() {

		return this.host;

	}

	public void setHost(final String host) {

		this.host = host;

	}

	public String getPostPath() {

		return this.postPath;

	}

	public void setPostPath(final String postPath) {

		this.postPath = postPath;

	}

	public String getAppToken() {

		return this.appToken;

	}

	public void setAppToken(final String token) {

		this.appToken = token;

	}

	public String getPreviewTitle() {

		return this.previewTitle;

	}

	public void setPreviewTitle(final String previewTitle) {

		this.previewTitle = previewTitle;

	}

	public String getPreviewDescription() {

		return this.previewDescription;

	}

	public void setPreviewDescription(final String previewDescription) {

		this.previewDescription = previewDescription;

	}

	public String getText() {

		return this.text;

	}

	public void setText(final String text) {

		this.text = text;

	}

	public String getHttpImageUrl() {

		return this.httpImageUrl;

	}

	public void setHttpImageUrl(final String httpImageUrl) {

		this.httpImageUrl = httpImageUrl;

	}

	public boolean isEnableNotification() {

		return this.enableNotification;

	}

	public void setEnableNotification(final boolean enablenotification) {

		this.enableNotification = enablenotification;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.enabling.ISLookUpService;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * Created by claudio on 20/06/16.

 */

public class LoadMappingProfileJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(LoadMappingProfileJobNode.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	private String profileId;

	private String loadedMappingParam;

	@Override

	protected String execute(final Env env) throws Exception {

		log.info(String.format("loading mapping from: %s", profileId));

		final String xquery = String.format("/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']",

				getProfileId());

		final String xslt = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);

		log.debug(String.format("mapping: %s", xslt));

		if (StringUtils.isBlank(xslt)) {

			throw new IllegalStateException("unable to load mapping from profile: " + getProfileId());

		}

		env.setAttribute(getLoadedMappingParam(), xslt);

		return Arc.DEFAULT_ARC;

	}

	public String getProfileId() {

		return profileId;

	}

	public void setProfileId(final String profileId) {

		this.profileId = profileId;

	}

	public String getLoadedMappingParam() {

		return loadedMappingParam;

	}

	public void setLoadedMappingParam(final String loadedMappingParam) {

		this.loadedMappingParam = loadedMappingParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to restore the cache of the portal specified by the additional BB parameter

 * <code>StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName()</code> to the last available backup.

 * <p>

 * To create a backup you must send a "backup" BB message.

 * </p>

 *

 * @author alessia

 * @see BackupStatsCacheJobNode

 */

public class RestoreStatsCacheJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(RestoreStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.RESTORE_CACHE.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to backup the stats cache of the portal specified by the additional BB parameter

 * <code>StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName()</code>. The cache can be restored with the last generated

 * backup by sending a "restore" BB message.

 *

 * @author alessia

 * @see RestoreStatsCacheJobNode

 */

public class BackupStatsCacheJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(BackupStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.BACKUP_CACHE.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to refresh the shadow cache according to the content of the shadow stats db for the portal specified by the

 * additional BB parameter <code>StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName()</code>.

 * <p>

 * In order for the given portal to actually use the generated cache, we must promote the cache with the BB message 'promoteCache'.

 * </p>

 *

 * @author alessia

 * @see eu.dnetlib.msro.workflows.nodes.stats.PromoteShadowCacheStatsJobNode

 */

public class RefreshShadowCacheJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(RefreshShadowCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to promote the shadow stats db and cache to public for the portal specified by the additional BB parameter

 * <code>StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName()</code>.

 * <p>

 * <p>

 * According to the StatsManagerService specifications in https://issue.openaire.research-infrastructures.eu/issues/914#note-49, this BB

 * action will:

 * <ol>

 * <li>promote the shadow cache to public</li>

 * <li>make a backup of the current stats db schema under the name 'backup'</li>

 * <li>replace it with the new one</li>

 * </ol>

 * </p>

 *

 * @author alessia

 */

public class PromoteShadowStatsJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(PromoteShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import eu.dnetlib.rmi.manager.MSROException;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to copy a cache of a portal to another portal cache.

 * <p>

 * The source portal is specified in the BB parameter <code>StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName()</code>.

 * </p>

 * <p>

 * The target portal is specified in the BB parameter <code>StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName()</code>

 * </p>

 *

 * @author alessia

 */

public class MigrateStatsCacheJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(MigrateStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * BB parameter defining which is the portal with the source cache to migrate.

	 */

	private String sourcePortal;

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.MIGRATE_CACHE.action());

		if (StringUtils.isBlank(this.sourcePortal)) { throw new MSROException("Parameter sourcePortal required to migrate the cache"); }

		job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName(), this.sourcePortal);

		final String targetPortal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(targetPortal)) {

			job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName(), targetPortal);

		} else {

			log.warn(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

	public String getSourcePortal() {

		return this.sourcePortal;

	}

	public void setSourcePortal(final String sourcePortal) {

		this.sourcePortal = sourcePortal;

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

public enum StatsManagerServiceBBAction {

	PROMOTE_SHADOW_STATS {

		@Override

		public String action() {

			return "promoteShadow";

		}

	},

	PROMOTE_SHADOW_CACHE {

		@Override

		public String action() {

			return "promoteCache";

		}

	},

	REFRESH_SHADOW_CACHE {

		@Override

		public String action() {

			return "refreshCache";

		}

	},

	VALIDATE_SHADOW_STATS {

		@Override

		public String action() {

			return "validate";

		}

	},

	BACKUP_CACHE {

		@Override

		public String action() {

			return "backup";

		}

	},

	RESTORE_CACHE {

		@Override

		public String action() {

			return "restore";

		}

	},

	MIGRATE_CACHE {

		@Override

		public String action() {

			return "migrate";

		}

		@Override

		public String getTargetPortalParamName() {

			return "targetCache";

		}

	};

	private static String DEFAULT_PORTAL_PARAM_NAME = "cache";

	public abstract String action();

	public String getSourcePortalParamName() {

		return DEFAULT_PORTAL_PARAM_NAME;

	}

	public String getTargetPortalParamName() {

		return DEFAULT_PORTAL_PARAM_NAME;

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to perform the content validation between the shadow stats db and the preproduction index (using the

 * prepublic search service) and creates a report, which is serialized in the blackboard job parameters.

 * <p>

 * The report is relative to the shadow stats db related to the portal specified by the additional BB parameter

 * <code>StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName()</code>

 * </p>

 *

 * @author alessia

 */

public class ValidateShadowStatsJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(ValidateShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Ask the StatsManagerService to promote the shadow cache so that its entries can be used by the portal specified by the additional BB

 * parameter <code>StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName()</code>.

 *

 * @author alessia

 */

public class PromoteShadowCacheStatsJobNode extends AbstractStatsJobNode {

	private static final Log log = LogFactory.getLog(PromoteShadowCacheStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.action());

		final String portal = getPortalName(token.getEnv());

		if (StringUtils.isNotBlank(portal)) {

			job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName(), portal);

		} else {

			log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import java.util.List;

import javax.annotation.Resource;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.enabling.ISLookUpException;

import eu.dnetlib.rmi.enabling.ISLookUpService;

import eu.dnetlib.rmi.manager.MSROException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

public class FindStatsServiceJobNode extends SimpleJobNode {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(FindStatsServiceJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Value(value = "${dnet.openaire.service.stats.lookup.xquery}")

	private String xquery;

	private String xqueryForServiceIdParam;

	/**

	 * {@inheritDoc}

	 *

	 * @throws ISLookUpException

	 * @throws MSROException

	 * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	public String execute(final Env env) throws ISLookUpException, MSROException {

		log.info("lookup for stats service: " + getXquery());

		final List<String> statsServiceIds = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());

		if (statsServiceIds.size() > 1) { throw new MSROException("too many stats services: " + statsServiceIds); }

		if (statsServiceIds.isEmpty()) {

			return "notFound";

		} else {

			env.setAttribute(getXqueryForServiceIdParam(), getXquery());

			return "found";

		}

	}

	public String getXquery() {

		return this.xquery;

	}

	public void setXquery(final String xquery) {

		this.xquery = xquery;

	}

	public String getXqueryForServiceIdParam() {

		return this.xqueryForServiceIdParam;

	}

	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {

		this.xqueryForServiceIdParam = xqueryForServiceIdParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.stats;

import java.util.List;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.enabling.ISLookUpException;

import eu.dnetlib.rmi.enabling.ISLookUpService;

import org.apache.commons.lang3.StringUtils;

/**

 * Abstract BBJob node for sending BB messages to the StatsManagerService. Subclasses must implement the prepareJob method.

 *

 * @author alessia

 */

public abstract class AbstractStatsJobNode extends BlackboardJobNode {

	private String xqueryForServiceIdParam;

	/**

	 * BB parameter defining which is the portal affected by the action. Overrides the value in the env parameter below.

	 */

	private String portal;

	/**

	 * Name of the env parameter where to find the target portal value. It is used only if the portal parameter above is not explicitly set

	 * in the workflow.

	 */

	private String portalParam;

	@Override

	protected String obtainServiceId(final Env env) {

		final String xquery = env.getAttribute(getXqueryForServiceIdParam(), String.class);

		List<String> statsServiceIds;

		try {

			statsServiceIds = getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(xquery);

			if (statsServiceIds.size() > 1) { throw new RuntimeException("Too many StatsManagerService ids found using query: " + xquery); }

			if (statsServiceIds.size() < 1) { throw new RuntimeException("StatsManagerService id not found using query: " + xquery); }

			return statsServiceIds.get(0);

		} catch (final ISLookUpException e) {

			throw new RuntimeException(e);

		}

	}

	public String getPortalName(final Env env) {

		if (StringUtils.isNotBlank(this.portal)) {

			return this.portal;

		} else {

			return env.getAttribute(this.portalParam, String.class);

		}

	}

	public String getXqueryForServiceIdParam() {

		return this.xqueryForServiceIdParam;

	}

	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {

		this.xqueryForServiceIdParam = xqueryForServiceIdParam;

	}

	public String getPortal() {

		return this.portal;

	}

	public void setPortal(final String portal) {

		this.portal = portal;

	}

	public String getPortalParam() {

		return this.portalParam;

	}

	public void setPortalParam(final String portalParam) {

		this.portalParam = portalParam;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.index.EntityGrouperConfigurationLoader;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.manager.MSROException;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

public class LoadEntityLinksJobNode extends SimpleJobNode {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(LoadEntityLinksJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String entityLinksParam;

	@Autowired

	private EntityGrouperConfigurationLoader grouperConfLoader;

	@Override

	protected String execute(final Env env) throws Exception {

		log.info("loading entity link table");

		final String links = this.grouperConfLoader.load();

		if (StringUtils.isBlank(links)) { throw new MSROException("empty links table"); }

		env.setAttribute(getEntityLinksParam(), links);

		return Arc.DEFAULT_ARC;

	}

	public String getEntityLinksParam() {

		return entityLinksParam;

	}

	public void setEntityLinksParam(final String entityLinksParam) {

		this.entityLinksParam = entityLinksParam;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import javax.annotation.Resource;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import eu.dnetlib.rmi.enabling.ISLookUpService;

public class ObtainOpenaireDataSourceParamsJobNode extends SimpleJobNode {

	private String providerId;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final Env env) throws Exception {

		final String query = "let $x := /*[.//RESOURCE_IDENTIFIER/@value='" + this.providerId + "']//EXTRA_FIELDS\n"

				+ "return concat($x/FIELD[./key='OpenAireDataSourceId']/value, ' @@@ ', $x/FIELD[./key='NamespacePrefix']/value)";

		final String[] arr = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query).split("@@@");

		final String origId = arr[0].trim();

		final String nsPrefix = arr[1].trim();

		// this is needed by the mdbuilder

		// TODO: update mdbuilder to use the env attributes below, whose names are defined in WorkflowConstants

		env.setAttribute("parentDatasourceId", origId);

		env.setAttribute("namespacePrefix", nsPrefix);

		env.setAttribute("dateOfCollection", DateUtils.now_ISO8601());

		// these are needed for validation and fill hostedby

		env.setAttribute(WorkflowsConstants.LOG_DATASOURCE_ID, origId);

		env.setAttribute(WorkflowsConstants.DATASOURCE_PREFIX, nsPrefix);

		return Arc.DEFAULT_ARC;

	}

	public String getProviderId() {

		return this.providerId;

	}

	public void setProviderId(final String providerId) {

		this.providerId = providerId;

	}

}

package eu.dnetlib.msro.workflows.nodes.utils;

import java.text.Normalizer;

/**

 * Created by claudio on 14/06/16.

 */

public class CleaningXsltFunctions {

	public static String clean(final String s) {

		return Normalizer.normalize(s, Normalizer.Form.NFD)

				.replaceAll("\\(.+\\)", "")

				.replaceAll("(\\W|\\p{InCombiningDiacriticalMarks}|\\p{Punct}|\\n|\\s)+", "")

				.toLowerCase()

				.trim();

	}

}

package eu.dnetlib.msro.workflows.nodes.utils;

import com.google.gson.Gson;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.collections.Pair;

public class Similarity {

	private Pair<String, String> pair;

	private Type type;

	public Similarity(final Pair<String, String> pair, final Type type) {

		super();

		this.setPair(pair);

		this.setType(type);

	}

	public Pair<String, String> getPair() {

		return pair;

	}

	public void setPair(final Pair<String, String> pair) {

		this.pair = pair;

	}

	public Type getType() {

		return type;

	}

	public void setType(final Type type) {

		this.type = type;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this, Similarity.class);

	}

}

package eu.dnetlib.msro.workflows.nodes.utils;

import java.util.List;

import com.google.common.collect.Lists;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.collections.Pair;

public class SimilarityMeshBuilder {

	public static List<Similarity> build(final Type type, final List<String> list) {

		return build(type, list, true);

	}

	public static List<Similarity> build(final Type type, final List<String> list, final boolean addTypePrefix) {

		final String typePrefix = String.valueOf(type.getNumber());

		final List<Similarity> res = Lists.newArrayList();

		for (int i = 0; i < list.size(); i++) {

			for (int j = 0; j < list.size(); j++) {

				if (!list.get(i).equals(list.get(j))) {

					final String source = addTypePrefix ? typePrefix + "|" + list.get(i) : list.get(i);

					final String target = addTypePrefix ? typePrefix + "|" + list.get(j) : list.get(j);

					res.add(new Similarity(new Pair<String, String>(source, target), type));

				}

			}

		}

		return res;

	}

}

package eu.dnetlib.msro.workflows.nodes.dedup;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;

import eu.dnetlib.msro.workflows.procs.Env;

import org.apache.commons.lang3.StringUtils;

import org.springframework.beans.factory.annotation.Autowired;

public class DedupConfigurationSetterJobNode extends AsyncJobNode {

	private String dedupConfigSequence;

	private String dedupConfigSequenceParam;

	private String entityTypeNameParam;

	private String entityTypeIdParam;

	@Autowired

	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;

	@Override

	protected String execute(final Env env) throws Exception {

		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("missing configuration sequence");

		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());

		env.setAttribute(getEntityTypeNameParam(), dedupOrchestration.getEntity().getName());

		env.setAttribute(getEntityTypeIdParam(), dedupOrchestration.getEntity().getCode());

		env.setAttribute(getDedupConfigSequenceParam(), dedupOrchestration);

		return Arc.DEFAULT_ARC;

	}

	public String getDedupConfigSequence() {

		return dedupConfigSequence;

	}

	public void setDedupConfigSequence(final String dedupConfigSequence) {

		this.dedupConfigSequence = dedupConfigSequence;

	}

	public String getDedupConfigSequenceParam() {

		return dedupConfigSequenceParam;

	}

	public void setDedupConfigSequenceParam(final String dedupConfigSequenceParam) {

		this.dedupConfigSequenceParam = dedupConfigSequenceParam;

	}

	public String getEntityTypeNameParam() {

		return entityTypeNameParam;

	}

	public void setEntityTypeNameParam(final String entityTypeNameParam) {

		this.entityTypeNameParam = entityTypeNameParam;

	}

	public String getEntityTypeIdParam() {

		return entityTypeIdParam;

	}

	public void setEntityTypeIdParam(final String entityTypeIdParam) {

		this.entityTypeIdParam = entityTypeIdParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.dedup;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;

import eu.dnetlib.msro.workflows.nodes.hadoop.SubmitHadoopJobNode;

import eu.dnetlib.msro.workflows.procs.Token;

import eu.dnetlib.pace.config.DedupConfig;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import static eu.dnetlib.msro.workflows.nodes.dedup.DedupConfigurationAwareJobNode.DEDUP_CONF;

public class DedupConfigurationLoaderJobNode extends SubmitHadoopJobNode {

	private static final Log log = LogFactory.getLog(DedupConfigurationLoaderJobNode.class);

	private String dedupConfigSequence;

	@Autowired

	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;

	@Override

	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {

		super.prepareJob(job, token);

		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());

		final DedupConfig currentConf = dedupOrchestration.getConfigurations().peek();

		log.debug("using dedup configuration: '" + currentConf + "'");

		job.getParameters().put(DEDUP_CONF, currentConf.toString());

	}

	public String getDedupConfigSequence() {

		return dedupConfigSequence;

	}

	public void setDedupConfigSequence(final String dedupConfigSequence) {

		this.dedupConfigSequence = dedupConfigSequence;

	}

}

package eu.dnetlib.msro.workflows.nodes.dedup;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.procs.Token;

import org.apache.commons.collections.CollectionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DedupDuplicateScanJobNode extends DedupConfigurationAwareJobNode {

	private static final Log log = LogFactory.getLog(DedupDuplicateScanJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {

		return new DedupBlackboardWorkflowJobListener(token);

	}

	private class DedupBlackboardWorkflowJobListener extends BlackboardWorkflowJobListener {

		public DedupBlackboardWorkflowJobListener(final Token token) {

			super(token);

		}

		@Override

		protected void onDone(final BlackboardJob job) {

			final DedupConfigurationOrchestration confs = getDedupConfigurationOrchestration();

			confs.getConfigurations().poll();

			log.info("checking dedup configs queue, size: " + confs.getConfigurations().size());

			if (CollectionUtils.isEmpty(confs.getConfigurations())) {

				log.info("dedup similarity scan done");

				getToken().release("done");

			} else {

				log.debug("remaining confs: " + confs);

				getToken().release(Arc.DEFAULT_ARC);

			}

		}

	}

}

package eu.dnetlib.msro.workflows.nodes.dedup;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;

import eu.dnetlib.msro.workflows.procs.Env;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * Created by claudio on 13/07/16.

 */

public class SelectWfPathByDedupEntityJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(SelectWfPathByDedupEntityJobNode.class);

	@Autowired

	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;

	private String dedupConfigSequence;

	@Override

	protected String execute(final Env env) throws Exception {

		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("missing configuration sequence");

		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());

		final String entityName = dedupOrchestration.getEntity().getName();

		log.info("following workflow path: " + entityName);

		return entityName;

	}

	public String getDedupConfigSequence() {

		return dedupConfigSequence;

	}

	public void setDedupConfigSequence(final String dedupConfigSequence) {

		this.dedupConfigSequence = dedupConfigSequence;

	}

}

package eu.dnetlib.msro.workflows.nodes.dedup;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import java.util.Queue;

import com.google.common.collect.Lists;

import com.google.common.collect.Queues;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.enabling.resultset.client.ResultSetClient;

import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;

import eu.dnetlib.msro.workflows.graph.Arc;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

import eu.dnetlib.msro.workflows.nodes.utils.Similarity;

import eu.dnetlib.msro.workflows.nodes.utils.SimilarityMeshBuilder;

import eu.dnetlib.msro.workflows.procs.Env;

import eu.dnetlib.rmi.common.ResultSet;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

public class BuildSimilarityMeshJobNode extends AsyncJobNode {

	private static final Log log = LogFactory.getLog(BuildSimilarityMeshJobNode.class);

	/**

	 * The result set client factory.

	 */

	@Autowired

	private ResultSetClient resultSetClient;

	@Autowired

	private ResultSetFactory resultSetFactory;

	private StringTemplate similarity;

	private String inputEprParam;

	private String outputEprParam;

	@Override

	protected String execute(final Env env) throws Exception {

		final ResultSet<?> inputRs = env.getAttribute(getInputEprParam(), ResultSet.class);

		final Iterator<String> it = resultSetClient.iter(inputRs, String.class).iterator();

		final Queue<Object> queue = Queues.newLinkedBlockingQueue();

		final SAXReader reader = new SAXReader();

		if (it.hasNext()) {

			populateQueue(queue, reader, it.next());

		}

		final ResultSet<String> rsOut = resultSetFactory.createResultSet(() -> {

			return new Iterator<String>() {

				@Override

				public boolean hasNext() {

					synchronized (queue) {

						return !queue.isEmpty();

					}

				}

				@Override

				public String next() {

					synchronized (queue) {

						final Object o = queue.poll();

						while (queue.isEmpty() && it.hasNext()) {

							populateQueue(queue, reader, it.next());

						}

						return buildSimilarity((Similarity) o);

					}

				}

				@Override

				public void remove() {

					throw new UnsupportedOperationException();

				}

			};

		});

		env.setAttribute(getOutputEprParam(), rsOut);

		return Arc.DEFAULT_ARC;

	}

	private void populateQueue(final Queue<Object> q, final SAXReader r, final String xml) {

		try {

			final Document d = r.read(new StringReader(xml));

			final String groupid = d.valueOf("//FIELD[@name='id']");

			final List<?> items = d.selectNodes("//FIELD[@name='group']/ITEM");

			final String entitytype = d.valueOf("//FIELD[@name='entitytype']");

			final List<String> group = Lists.newArrayList();