Project

General

Profile

« Previous | Next » 

Revision 56987

Added by Alessia Bardi over 4 years ago

refactoring for proper publishing on JRR

View differences:

modules/dnet-parthenos-publisher/trunk/src/main/java/eu/dnetlib/parthenos/virtuoso/VirtuosoReadAPI.java
4 4 
import java.io.OutputStream;
5 5 
import java.io.StringWriter;
6 6 
import java.util.HashMap;
7 
import java.util.Iterator;
8 7 
import java.util.List;
9 8 
import java.util.Map;
10 9 

  		




  ......




  74
  73
  
    
		log.debug("SPARQL query: "+q);


  		




  75
  74
  
    
		final QueryEngineHTTP serviceRequest = new QueryEngineHTTP(sparqlUrl, q);


  		




  76
  75
  
    
		ResultSet subjects = serviceRequest.execSelect();


  		




  77
  
  
    
		Iterator<String> s = Iterators.transform(subjects, qs -> qs.getResource("s").getURI());


  		




  78
  
  
    
		List<String> res = Lists.newArrayList(s);


  		




  
  76
  
    
		List<String> res = Lists.newArrayList(Iterators.filter(Iterators.transform(subjects, qs -> qs.getResource("s").getURI()), input -> input.startsWith("http")));


  		




  79
  77
  
    
		serviceRequest.close();


  		




  80
  78
  
    
		return res;


  		




  81
  79
  
    
	}


  		




  ......




  85
  83
  
    
	@ResponseStatus(value = HttpStatus.OK)


  		




  86
  84
  
    
	public void getSubject(@RequestParam final String subjectURL, @RequestParam final String typeName, @RequestParam(name="timeout") final String timeoutMs, final OutputStream responseStream)


  		




  87
  85
  
    
			throws IOException, TemplateException, ParthenosPublisherException {


  		




  
  86
  
    

  		




  
  87
  
    
		sendConstructResponse(generateQuery(subjectURL, typeName), timeoutMs, responseStream);


  		




  
  88
  
    
	}


  		




  
  89
  
    

  		




  
  90
  
    
	public String getRDF(final String subjectURL, final String typeName, final String apiId, final String timeoutMs)


  		




  
  91
  
    
			throws IOException, TemplateException, ParthenosPublisherException {


  		




  
  92
  
    

  		




  
  93
  
    
		return executeSparqlPost(generateQuery(subjectURL, typeName), timeoutMs);


  		




  
  94
  
    
	}


  		




  
  95
  
    

  		




  
  96
  
    
	protected String generateQuery(final String subjectURL, final String typeName) throws IOException, TemplateException {


  		




  88
  97
  
    
		String templateName = typeName+".sparql";


  		




  89
  98
  
    
		Template temp = freemarkerConfig.getTemplate(templateName);


  		




  90
  99
  
    
		Map<String, String> values = new HashMap<>();


  		




  ......




  94
  103
  
    
		temp.process(values, sw);


  		




  95
  104
  
    
		String q = sw.toString();


  		




  96
  105
  
    
		log.debug("Querying for "+subjectURL+" with query "+templateName);


  		




  97
  
  
    
		sendConstructResponse(q, timeoutMs, responseStream);


  		




  
  106
  
    
		return q;


  		




  98
  107
  
    
	}


  		




  99
  108
  
    

  		




  100
  109
  
    

  		












  

    
      modules/dnet-parthenos-publisher/trunk/src/main/java/eu/dnetlib/parthenos/jrr/JRRPublisherHelper.java
    
  





  
  1
  
    
package eu.dnetlib.parthenos.jrr;


  		




  
  2
  
    

  		




  
  3
  
    
import java.util.List;


  		




  
  4
  
    

  		




  
  5
  
    
import eu.dnetlib.parthenos.virtuoso.VirtuosoReadAPI;


  		




  
  6
  
    
import org.apache.commons.logging.Log;


  		




  
  7
  
    
import org.apache.commons.logging.LogFactory;


  		




  
  8
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  
  9
  
    
import org.springframework.stereotype.Component;


  		




  
  10
  
    

  		




  
  11
  
    
/**


  		




  
  12
  
    
 * Created by Alessia Bardi on 2019-06-26.


  		




  
  13
  
    
 *


  		




  
  14
  
    
 * @author Alessia Bardi


  		




  
  15
  
    
 */


  		




  
  16
  
    
@Component


  		




  
  17
  
    
public class JRRPublisherHelper {


  		




  
  18
  
    

  		




  
  19
  
    
	private static final Log log = LogFactory.getLog(JRRPublisherHelper.class);


  		




  
  20
  
    

  		




  
  21
  
    
	protected final static int LIMIT = 100;


  		




  
  22
  
    
	protected static final String ANY_TIME_QUERY_MS = "1800000"; //1800000 == 3 mins


  		




  
  23
  
    

  		




  
  24
  
    
	@Autowired


  		




  
  25
  
    
	private VirtuosoReadAPI virtuosoReadAPI;


  		




  
  26
  
    
	@Autowired


  		




  
  27
  
    
	private JRRPublisher jrrPublisher;


  		




  
  28
  
    

  		




  
  29
  
    

  		




  
  30
  
    
	public int publish(final String typeNamespace, final String typeName, final String apiId, final String datasourceName) {


  		




  
  31
  
    
		int offset = 0;


  		




  
  32
  
    
		boolean again;


  		




  
  33
  
    
		int countSuccess = 0;


  		




  
  34
  
    
		int countFailed = 0;


  		




  
  35
  
    
		do {


  		




  
  36
  
    
			//get list of subjects of the entityType in the apiURI graph


  		




  
  37
  
    
			List<String> subjects = virtuosoReadAPI.getSubjectsForApiWithType(apiId, typeNamespace, typeName, LIMIT, offset);


  		




  
  38
  
    
			//get the RDF


  		




  
  39
  
    
			for(String subject : subjects) {


  		




  
  40
  
    
				try {


  		




  
  41
  
    
					String fullTypeName = typeNamespace + typeName;


  		




  
  42
  
    
					String record = virtuosoReadAPI.getRDF(subject, fullTypeName, apiId, ANY_TIME_QUERY_MS);


  		




  
  43
  
    
					//request publishing on JRR (need to have the OAI header with the proper subject as objIdentifier


  		




  
  44
  
    
					jrrPublisher.register(record, subject, datasourceName);


  		




  
  45
  
    
					countSuccess++;


  		




  
  46
  
    
				}catch(Throwable t){


  		




  
  47
  
    
					log.warn(String.format("Cannot publish %s with URL %s", typeName, subject));


  		




  
  48
  
    
					log.warn(t.getMessage());


  		




  
  49
  
    
					countFailed++;


  		




  
  50
  
    
				}


  		




  
  51
  
    
			}


  		




  
  52
  
    
			again = subjects.size() == LIMIT;


  		




  
  53
  
    
			offset += LIMIT;


  		




  
  54
  
    
		} while(again);


  		




  
  55
  
    
		log.info(String.format("%s : published %d, failed %d", typeName, countSuccess, countFailed));


  		




  
  56
  
    
		return countSuccess;


  		




  
  57
  
    
	}


  		




  
  58
  
    

  		




  
  59
  
    
	public JRRPublisherHelper(){


  		




  
  60
  
    

  		




  
  61
  
    
	}


  		




  
  62
  
    

  		




  
  63
  
    
	public VirtuosoReadAPI getVirtuosoReadAPI() {


  		




  
  64
  
    
		return virtuosoReadAPI;


  		




  
  65
  
    
	}


  		




  
  66
  
    

  		




  
  67
  
    
	public void setVirtuosoReadAPI(final VirtuosoReadAPI virtuosoReadAPI) {


  		




  
  68
  
    
		this.virtuosoReadAPI = virtuosoReadAPI;


  		




  
  69
  
    
	}


  		




  
  70
  
    

  		




  
  71
  
    
	public JRRPublisher getJrrPublisher() {


  		




  
  72
  
    
		return jrrPublisher;


  		




  
  73
  
    
	}


  		




  
  74
  
    

  		




  
  75
  
    
	public void setJrrPublisher(final JRRPublisher jrrPublisher) {


  		




  
  76
  
    
		this.jrrPublisher = jrrPublisher;


  		




  
  77
  
    
	}


  		




  
  78
  
    
}


  		












  

    
      modules/dnet-parthenos-publisher/trunk/src/main/java/eu/dnetlib/parthenos/publisher/ParthenosPublisherController.java
    
  





  1
  1
  
    
package eu.dnetlib.parthenos.publisher;


  		




  2
  2
  
    

  		




  
  3
  
    
import java.io.IOException;


  		




  
  4
  
    
import java.net.URISyntaxException;


  		




  
  5
  
    

  		




  
  6
  
    
import eu.dnetlib.parthenos.jrr.JRRPublisherHelper;


  		




  3
  7
  
    
import eu.dnetlib.parthenos.publisher.ParthenosPublisherHelper.ParthenosTargets;


  		




  
  8
  
    
import freemarker.template.TemplateException;


  		




  4
  9
  
    
import org.apache.commons.logging.Log;


  		




  5
  10
  
    
import org.apache.commons.logging.LogFactory;


  		




  6
  11
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  ......




  18
  23
  
    
	@Autowired


  		




  19
  24
  
    
	private ParthenosPublisherHelper parthenosPublisherHelper;


  		




  20
  25
  
    

  		




  
  26
  
    
	@Autowired


  		




  
  27
  
    
	private JRRPublisherHelper jrrPublisherHelper;


  		




  
  28
  
    

  		




  
  29
  
    
	@RequestMapping(value="publishJRR", method = RequestMethod.POST)


  		




  
  30
  
    
	public int publishJRR(@RequestParam final String typeNamespace, @RequestParam final String typeName, @RequestParam final String datasourceApi, @RequestParam final String datasourceName)


  		




  
  31
  
    
			throws URISyntaxException, TemplateException, IOException, InterruptedException, ParthenosPublisherException {


  		




  
  32
  
    
		return jrrPublisherHelper.publish(typeNamespace, typeName, datasourceApi, datasourceName);


  		




  
  33
  
    
	}


  		




  
  34
  
    

  		




  21
  35
  
    
	@RequestMapping(value = "/publish", method = RequestMethod.POST)


  		




  22
  36
  
    
	public void publish(@RequestParam final String record, @RequestParam(required = false) String parthenosTarget) throws ParthenosPublisherException {


  		




  23
  37
  
    
		getParthenosPublisherHelper().publish(record, getTarget(parthenosTarget));


  		












  

    
      modules/dnet-parthenos-publisher/trunk/pom.xml
    
  





  11
  11
  
    
	<groupId>eu.dnetlib</groupId>


  		




  12
  12
  
    
	<artifactId>dnet-parthenos-publisher</artifactId>


  		




  13
  13
  
    
	<packaging>jar</packaging>


  		




  14
  
  
    
	<version>3.0.0-SNAPSHOT</version>


  		




  
  14
  
    
	<version>4.0.0-SNAPSHOT</version>


  		




  15
  15
  
    
	<scm>


  		




  16
  16
  
    
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-parthenos-publisher/trunk</developerConnection>


  		




  17
  17
  
    
		<url>https://github.com/spring-projects/spring-boot/spring-boot-starter-parent/dnet-parthenos-publisher</url>


  		












Also available in:  Unified diff