/ - Diff - D-Net - D-Net project tracking tool

     package eu.dnetlib;
     import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
     import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
     import org.springframework.beans.factory.annotation.Value;
     import org.springframework.context.annotation.Bean;
     import org.springframework.context.annotation.Configuration;
     /**
      * Created by claudio on 04/07/2017.
      */
     @Configuration
     public class DNetConfig {
     	@Value("${services.is.lookup.url}")
     	private String isLookupUrl;
     	@Bean
     	public ISLookUpService getLookUpService() {
     		final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
     		jaxWsProxyFactory.setServiceClass(ISLookUpService.class);
     		jaxWsProxyFactory.setAddress(isLookupUrl);
     		return (ISLookUpService) jaxWsProxyFactory.create();
+    	}
+    }

     package eu.dnetlib.openaire.exporter.model.project;
     import java.io.IOException;
     import java.io.StringReader;
     import java.io.StringWriter;
     import java.util.List;
     import com.google.gson.Gson;
     import org.supercsv.cellprocessor.Optional;
     import org.supercsv.cellprocessor.ift.CellProcessor;
     import org.supercsv.cellprocessor.ift.StringCellProcessor;
     import org.supercsv.io.CsvBeanReader;
     import org.supercsv.io.CsvBeanWriter;
     import org.supercsv.io.ICsvBeanReader;
     import org.supercsv.io.ICsvBeanWriter;
     import org.supercsv.prefs.CsvPreference;
     import org.supercsv.util.CsvContext;
     /**
      * Created by claudio on 22/09/16.
      */
     public class ProjectDetail {
     	private static final String[] NAMEMAPPING = { "projectId", "acronym", "code", "jsonextrainfo", "fundingPath" };
     	private String projectId;
     	private String acronym;
     	private String code;
     	private String jsonextrainfo;
     	private List<String> fundingPath;
     	public ProjectDetail() {}
     	public static ProjectDetail fromJson(final String json) {
     		return new Gson().fromJson(json, ProjectDetail.class);
+    	}
     	public static ProjectDetail fromCSV(final String csv) throws IOException {
     		ICsvBeanReader beanReader = null;
     		try {
     			beanReader = new CsvBeanReader(new StringReader(csv), CsvPreference.STANDARD_PREFERENCE);
     			return beanReader.read(ProjectDetail.class, NAMEMAPPING, getProcessors(new StringCellProcessor() {
     				@Override
     				public Object execute(final Object value, final CsvContext context) {
     					return new Gson().fromJson(value.toString(), List.class);
+    				}
     			}));
     		} finally {
     			if (beanReader != null) {
     				beanReader.close();
+    			}
+    		}
+    	}
     	/**
     	 * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. Empty
     	 * columns are read as null (hence the NotNull() for mandatory columns).
+    	 *
     	 * @return the cell processors
     	 */
     	private static CellProcessor[] getProcessors(final CellProcessor fundingPathProcessor) {
     		return new CellProcessor[] {
     				new Optional(), // projectId
     				new Optional(), // acronym
     				new Optional(), // code
     				new Optional(), // jsonextrainfo
     				fundingPathProcessor
     		};
+    	}
     	public String asJson() {
     		return new Gson().toJson(this) + '\n';
+    	}
     	public String asCSV() throws IOException {
     		final StringWriter sb = new StringWriter();
     		try (ICsvBeanWriter beanWriter = new CsvBeanWriter(sb, CsvPreference.STANDARD_PREFERENCE)) {
     			beanWriter.write(this, NAMEMAPPING, getProcessors(new StringCellProcessor() {
     				@Override
     				public Object execute(final Object value, final CsvContext context) {
     					return new Gson().toJson(value);
+    				}
     			}));
     			beanWriter.flush();
+    		}
     		return sb.toString();
+    	}
     	public String getProjectId() {
     		return projectId;
+    	}
     	public ProjectDetail setProjectId(final String projectId) {
     		this.projectId = projectId;
     		return this;
+    	}
     	public String getAcronym() {
     		return acronym;
+    	}
     	public ProjectDetail setAcronym(final String acronym) {
     		this.acronym = acronym;
     		return this;
+    	}
     	public String getCode() {
     		return code;
+    	}
     	public ProjectDetail setCode(final String code) {
     		this.code = code;
     		return this;
+    	}
     	public String getJsonextrainfo() {
     		return jsonextrainfo;
+    	}
     	public ProjectDetail setJsonextrainfo(final String jsonextrainfo) {
     		this.jsonextrainfo = jsonextrainfo;
     		return this;
+    	}
     	public List<String> getFundingPath() {
     		return fundingPath;
+    	}
     	public ProjectDetail setFundingPath(final List<String> fundingPath) {
     		this.fundingPath = fundingPath;
     		return this;
+    	}
+    }

modules/dnet-openaire-exporter/trunk/src/main/resources/eu/dnetlib/openaire/exporter/sql/projects_details.sql
1		SELECT * FROM projects_details;

     package eu.dnetlib.openaire.exporter.model.project;
     import java.io.IOException;
     import org.junit.Test;
     /**
      * Created by claudio on 05/07/2017.
      */
     public class ProjectDetailsTest {
     	@Test
     	public void testCSV() throws IOException {
     		final ProjectDetails p = ProjectDetails.fromCSV(
     				"arc_________::ANZCCART,,ANZCCART,{},\"[\"\"\\u003cfundingtree\\u003e\\n      \\u003cfunder\\u003e\\n         \\u003cid\\u003earc_________::ARC\\u003c/id\\u003e\\n         \\u003cshortname\\u003eARC\\u003c/shortname\\u003e\\n         \\u003cname\\u003eAustralian Research Council (ARC)\\u003c/name\\u003e\\n         \\u003cjurisdiction\\u003eAU\\u003c/jurisdiction\\u003e\\n      \\u003c/funder\\u003e\\n      \\u003cfunding_level_0\\u003e\\n         \\u003cid\\u003earc_________::ARC::Special Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/id\\u003e\\n         \\u003cname\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/name\\u003e\\n         \\u003cdescription\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/description\\u003e\\n         \\u003cparent/\\u003e\\n         \\u003cclass\\u003earc:fundingStream\\u003c/class\\u003e\\n      \\u003c/funding_level_0\\u003e\\n   \\u003c/fundingtree\\u003e\"\"]\"");
     		System.out.println(p.asJson());
     		System.out.println(p.asCSV());
+    	}
+    }

     import com.mongodb.MongoClient;
     import com.mongodb.MongoClientOptions;
     import com.mongodb.ServerAddress;
     import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
     import org.apache.commons.dbcp2.BasicDataSource;
     import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
     import org.springframework.beans.factory.annotation.Value;
     import org.springframework.context.annotation.Bean;
     import org.springframework.context.annotation.Configuration;
-...
     	@Value("${openaire.datasource.publisher.mongodb.connectionsPerHost}")
     	private int connectionsPerHost;
     	@Value("${services.is.lookup.url}")
     	private String isLookupUrl;
     	@Bean
     	public ISLookUpService getLookUpService() {
     		final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
     		jaxWsProxyFactory.setServiceClass(ISLookUpService.class);
     		jaxWsProxyFactory.setAddress(isLookupUrl);
     		return (ISLookUpService) jaxWsProxyFactory.create();
+    	}
     	@Bean
     	public BasicDataSource getProjectDataSource() {
     		return getDatasource(driverClassName, jdbcUrl, jdbcUser, jdbcPwd, jdbcMinIdle, jdbcMaxidle);
+    	}

     package eu.dnetlib.openaire.exporter.model.project;
     import java.sql.Date;
     import java.util.List;
     import javax.persistence.Column;
     import javax.persistence.Entity;
     import javax.persistence.Id;
     import javax.persistence.Table;
     import com.fasterxml.jackson.annotation.JsonIgnore;
     import com.google.common.collect.Lists;
     import io.swagger.annotations.ApiModel;
     import org.apache.commons.lang3.StringUtils;
     /**
      * Created by claudio on 05/07/2017.
      */
     @Entity
     @Table(name = "projects_tsv")
     @ApiModel(value = "Project TSV model", description = "project TSV model description")
     public class ProjectTsv {
     	@Id
     	@JsonIgnore
     	private int id;
     	private String code;
     	private String acronym;
     	private String title;
     	@Column(name = "call_identifier")
     	private String callIdentifier;
     	private Date startdate;
     	private Date enddate;
     	@Column(name = "ec_sc39")
     	private Boolean ecSc39;
     	@Column(name = "oa_mandate_for_publications")
     	private Boolean oaMandateForPublications;
     	@Column(name = "oa_mandate_for_datasets")
     	private Boolean oaMandateForDatasets;
     	@JsonIgnore
     	private String fundingpathid;
     	private String description;
     	@Column(name = "legalname")
     	private String orgLegalname;
     	@Column(name = "country")
     	private String orgCountry;
     	@Column(name = "role")
     	private String orgRole;
     	private String firstname;
     	private String secondnames;
     	private String email;
     	public ProjectTsv() {}
     	public List<String> asList() {
     		return Lists.newArrayList(
     				clean(getCode()),
     				clean(getAcronym()),
     				clean(getTitle()),
     				clean(getCallIdentifier()),
     				clean(getStartdate() != null ? getStartdate().toString() : ""),
     				clean(getEnddate() != null ? getEnddate().toString() : ""),
     				clean(String.valueOf(isOaMandateForPublications())),
     				clean(String.valueOf(isOaMandateForDatasets())),
     				clean(getDescription()),
     				clean(getOrgLegalname()),
     				clean(getOrgCountry()),
     				clean(getOrgRole()),
     				clean(getFirstname()),
     				clean(getSecondnames()),
     				clean(getEmail()));
+    	}
     	private String clean(final String s) {
     		return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : "";
+    	}
     	public int getId() {
     		return id;
+    	}
     	public void setId(final int id) {
     		this.id = id;
+    	}
     	public String getCode() {
     		return code;
+    	}
     	public void setCode(final String code) {
     		this.code = code;
+    	}
     	public String getAcronym() {
     		return acronym;
+    	}
     	public void setAcronym(final String acronym) {
     		this.acronym = acronym;
+    	}
     	public String getTitle() {
     		return title;
+    	}
     	public void setTitle(final String title) {
     		this.title = title;
+    	}
     	public String getCallIdentifier() {
     		return callIdentifier;
+    	}
     	public void setCallIdentifier(final String callIdentifier) {
     		this.callIdentifier = callIdentifier;
+    	}
     	public Date getStartdate() {
     		return startdate;
+    	}
     	public void setStartdate(final Date startdate) {
     		this.startdate = startdate;
+    	}
     	public Date getEnddate() {
     		return enddate;
+    	}
     	public void setEnddate(final Date enddate) {
     		this.enddate = enddate;
+    	}
     	public Boolean isEcSc39() {
     		return ecSc39;
+    	}
     	public void setEcSc39(final Boolean ecSc39) {
     		this.ecSc39 = ecSc39;
+    	}
     	public Boolean isOaMandateForPublications() {
     		return oaMandateForPublications;
+    	}
     	public void setOaMandateForPublications(final Boolean oaMandateForPublications) {
     		this.oaMandateForPublications = oaMandateForPublications;
+    	}
     	public Boolean isOaMandateForDatasets() {
     		return oaMandateForDatasets;
+    	}
     	public void setOaMandateForDatasets(final Boolean oaMandateForDatasets) {
     		this.oaMandateForDatasets = oaMandateForDatasets;
+    	}
     	public String getFundingpathid() {
     		return fundingpathid;
+    	}
     	public void setFundingpathid(final String fundingpathid) {
     		this.fundingpathid = fundingpathid;
+    	}
     	public String getDescription() {
     		return description;
+    	}
     	public void setDescription(final String description) {
     		this.description = description;
+    	}
     	public String getOrgLegalname() {
     		return orgLegalname;
+    	}
     	public void setOrgLegalname(final String orgLegalname) {
     		this.orgLegalname = orgLegalname;
+    	}
     	public String getOrgCountry() {
     		return orgCountry;
+    	}
     	public void setOrgCountry(final String orgCountry) {
     		this.orgCountry = orgCountry;
+    	}
     	public String getOrgRole() {
     		return orgRole;
+    	}
     	public void setOrgRole(final String orgRole) {
     		this.orgRole = orgRole;
+    	}
     	public String getFirstname() {
     		return firstname;
+    	}
     	public void setFirstname(final String firstname) {
     		this.firstname = firstname;
+    	}
     	public String getSecondnames() {
     		return secondnames;
+    	}
     	public void setSecondnames(final String secondnames) {
     		this.secondnames = secondnames;
+    	}
     	public String getEmail() {
     		return email;
+    	}
     	public void setEmail(final String email) {
     		this.email = email;
+    	}
+    }

     package eu.dnetlib.openaire.exporter.model.project;
     import java.io.IOException;
     import java.io.StringReader;
     import java.io.StringWriter;
     import java.util.Arrays;
     import java.util.List;
     import javax.persistence.*;
     import com.fasterxml.jackson.annotation.JsonIgnore;
     import com.google.gson.Gson;
     import io.swagger.annotations.ApiModel;
     import org.hibernate.annotations.Type;
     import org.supercsv.cellprocessor.Optional;
     import org.supercsv.cellprocessor.ift.CellProcessor;
     import org.supercsv.cellprocessor.ift.StringCellProcessor;
     import org.supercsv.io.CsvBeanReader;
     import org.supercsv.io.CsvBeanWriter;
     import org.supercsv.io.ICsvBeanReader;
     import org.supercsv.io.ICsvBeanWriter;
     import org.supercsv.prefs.CsvPreference;
     import org.supercsv.util.CsvContext;
-...
     /**
      * Created by claudio on 04/07/2017.
      */
     //@Entity
     //@Table(name = "project_details")
     //@ApiModel(value = "Project details model", description = "provides project details")
     @Entity
     @Table(name = "project_details")
     @ApiModel(value = "Project details model", description = "provides project details")
     public class ProjectDetails {
     	private static final String[] NAMEMAPPING = { "projectid", "acronym", "code", "jsonextrainfo", "fundingpath" };
     	@Transient
     	@JsonIgnore
     	private transient static final String[] FIELDS = Arrays.stream(ProjectDetails.class.getDeclaredFields())
     			.map(f -> f.getName())
     			.filter(s -> !s.equals("FIELDS"))
     			.filter(s -> !s.startsWith("optional"))
     			.toArray(String[]::new);
     	//@Id
     	private String projectid;
     	@Id
     	@Column(name = "projectid")
     	private String projectId;
     	private String acronym;
     	private String code;
     	private String optional1;
     	private String optional2;
     	@Transient
     	@JsonIgnore
     	private transient String optional1;
     	@Transient
     	@JsonIgnore
     	private transient String optional2;
     	private String jsonextrainfo;
     	private String[] fundingpath;
     	@Type(type = "eu.dnetlib.openaire.exporter.model.GenericArrayUserType")
     	@Column(name = "fundingpath", columnDefinition = "text[]")
     	private String[] fundingPath;
     	public ProjectDetails() {
+    	}
     	public String getProjectid() {
     		return projectid;
     	public String getProjectId() {
     		return projectId;
+    	}
     	public void setProjectid(final String projectid) {
     		this.projectid = projectid;
     	public void setProjectId(final String projectId) {
     		this.projectId = projectId;
+    	}
     	public String getAcronym() {
-...
     		this.jsonextrainfo = jsonextrainfo;
+    	}
     	public String[] getFundingpath() {
     		return fundingpath;
     	public String[] getFundingPath() {
     		return fundingPath;
+    	}
     	public void setFundingpath(final String[] fundingpath) {
     		this.fundingpath = fundingpath;
     	public void setFundingPath(final String[] fundingPath) {
     		this.fundingPath = fundingPath;
+    	}
     	public static ProjectDetails fromJson(final String json) {
     		return new Gson().fromJson(json, ProjectDetails.class);
+    	}
     	public static ProjectDetails fromCSV(final String csv) throws IOException {
     		ICsvBeanReader beanReader = null;
     		try {
     			beanReader = new CsvBeanReader(new StringReader(csv), CsvPreference.STANDARD_PREFERENCE);
     			return beanReader.read(ProjectDetails.class, FIELDS, getProcessors(new StringCellProcessor() {
     				@Override
     				public Object execute(final Object value, final CsvContext context) {
     					return new Gson().fromJson(value.toString(), List.class);
+    				}
     			}));
     		} finally {
     			if (beanReader != null) {
     				beanReader.close();
+    			}
+    		}
+    	}
     	/**
     	 * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. Empty
     	 * columns are read as null (hence the NotNull() for mandatory columns).
-...
     	public String asCSV() throws IOException {
     		final StringWriter sb = new StringWriter();
     		try (ICsvBeanWriter beanWriter = new CsvBeanWriter(sb, CsvPreference.STANDARD_PREFERENCE)) {
     			beanWriter.write(this, NAMEMAPPING, getProcessors(new StringCellProcessor() {
     			beanWriter.write(this, FIELDS, getProcessors(new StringCellProcessor() {
     				@Override
     				public Object execute(final Object value, final CsvContext context) {
     					return new Gson().toJson(value);

     package eu.dnetlib.openaire.exporter.model;
     import java.sql.Array;
     import java.sql.Connection;
     import java.sql.SQLException;
     import java.util.Arrays;
     import java.util.List;
     import java.util.Map;
     import java.util.stream.Collectors;
     import javax.persistence.AttributeConverter;
     import javax.persistence.Converter;
     import javax.sql.DataSource;
     import org.springframework.beans.BeansException;
     import org.springframework.context.ApplicationContext;
     import org.springframework.context.ApplicationContextAware;
     /**
      * Created by claudio on 05/07/2017.
      */
     @Converter
     public class ConverterTextArray implements AttributeConverter<List<String>, Array>, ApplicationContextAware {
     	private ApplicationContext applicationContext;
     	@Override
     	public Array convertToDatabaseColumn(List<String> attribute) {
     		final Map<String, DataSource> datasources = applicationContext.getBeansOfType(DataSource.class);
     		DataSource source = datasources.values().stream().findFirst().get();
     		try {
     			Connection conn = source.getConnection();
     			Array array = conn.createArrayOf("text", attribute.toArray());
     			return  array;
     		} catch (SQLException e) {
     			e.printStackTrace();
+    		}
     		return null;
+    	}
     	@Override
     	public List<String> convertToEntityAttribute(Array dbData) {
     		try {
     			return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList());
     		} catch (SQLException e) {
     			e.printStackTrace();
     			return null;
+    		}
+    	}
     	@Override
     	public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
     		this.applicationContext = applicationContext;
+    	}
+    }

     package eu.dnetlib.openaire.exporter.model;
     import java.io.Serializable;
     import java.sql.*;
     import org.hibernate.HibernateException;
     import org.hibernate.engine.spi.SessionImplementor;
     import org.hibernate.usertype.UserType;
     /**
      * Created by claudio on 05/07/2017.
      */
     public class GenericArrayUserType<T extends Serializable> implements UserType {
     	protected static final int[] SQL_TYPES = { Types.ARRAY };
     	private  Class<T> typeParameterClass;
     	@Override
     	public Object assemble(Serializable cached, Object owner) throws HibernateException {
     		return this.deepCopy(cached);
+    	}
     	@Override
     	public Object deepCopy(Object value) throws HibernateException {
     		return value;
+    	}
     	@SuppressWarnings("unchecked")
     	@Override
     	public Serializable disassemble(Object value) throws HibernateException {
     		return (T) this.deepCopy(value);
+    	}
     	@Override
     	public boolean equals(Object x, Object y) throws HibernateException {
     		if (x == null) {
     			return y == null;
+    		}
     		return x.equals(y);
+    	}
     	@Override
     	public int hashCode(Object x) throws HibernateException {
     		return x.hashCode();
+    	}
     	@Override
     	public boolean isMutable() {
     		return true;
+    	}
     	@Override
     	public Object nullSafeGet(ResultSet resultSet, String[] names, SessionImplementor session, Object owner)
     			throws HibernateException, SQLException {
     		if (resultSet.wasNull()) {
     			return null;
+    		}
     		if (resultSet.getArray(names[0]) == null) {
     			return new Integer[0];
+    		}
     		Array array = resultSet.getArray(names[0]);
     		@SuppressWarnings("unchecked")
     		T javaArray = (T) array.getArray();
     		return javaArray;
+    	}
     	@Override
     	public void nullSafeSet(PreparedStatement statement, Object value, int index, SessionImplementor session)
     			throws HibernateException, SQLException {
     		Connection connection = statement.getConnection();
     		if (value == null) {
     			statement.setNull(index, SQL_TYPES[0]);
     		} else {
     			@SuppressWarnings("unchecked")
     			T castObject = (T) value;
     			Array array = connection.createArrayOf("integer", (Object[]) castObject);
     			statement.setArray(index, array);
+    		}
+    	}
     	@Override
     	public Object replace(Object original, Object target, Object owner) throws HibernateException {
     		return original;
+    	}
     	@Override
     	public Class<T> returnedClass() {
     		return typeParameterClass;
+    	}
     	@Override
     	public int[] sqlTypes() {
     		return new int[] { Types.ARRAY };
+    	}
+    }

     package eu.dnetlib.openaire.exporter.project.repository;
     import eu.dnetlib.openaire.exporter.model.project.ProjectTsv;
     import org.springframework.data.repository.PagingAndSortingRepository;
     import org.springframework.stereotype.Repository;
     /**
      * Created by claudio on 04/07/2017.
      */
     @Repository
     public interface ProjectTsvRepository extends PagingAndSortingRepository<ProjectTsv, String> {
     	Iterable<ProjectTsv> findByFundingpathidStartingWithOrderByAcronym(String fundingpathid);
     	Iterable<ProjectTsv> findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(String fundingpathid, boolean article293);
+    }

     package eu.dnetlib.openaire.exporter.project.repository;
     import eu.dnetlib.openaire.exporter.model.project.ProjectDetails;
     import org.springframework.data.repository.PagingAndSortingRepository;
     import org.springframework.stereotype.Repository;
     /**
      * Created by claudio on 04/07/2017.
      */
     //@Repository
     public interface ProjectDetailsRepository /*extends PagingAndSortingRepository<ProjectDetails, String>*/ {
     @Repository
     public interface ProjectDetailsRepository extends PagingAndSortingRepository<ProjectDetails, String> {
+    }

     import java.time.LocalDateTime;
     import java.util.Arrays;
     import java.util.List;
     import java.util.zip.GZIPOutputStream;
     import java.util.zip.ZipOutputStream;
     import com.google.common.base.Function;
-...
     import com.google.common.collect.Iterables;
     import com.google.common.collect.Lists;
     import eu.dnetlib.openaire.exporter.model.project.Project;
     import eu.dnetlib.openaire.exporter.model.project.ProjectDetail;
     import org.antlr.stringtemplate.StringTemplate;
     import org.apache.commons.dbcp2.BasicDataSource;
     import org.apache.commons.lang3.StringUtils;
-...
+    	}
     	public void streamProjectDetails(final String sql, final OutputStream out, final String format) throws SQLException, IOException {
     		if (log.isDebugEnabled()) {
     			log.debug("Thread " + Thread.currentThread().getId() + " begin");
+    		}
     		final LocalDateTime start = LocalDateTime.now();
     		int i = 0;
     		try(final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) {
     			while (rs.next()) {
     				final ProjectDetail p = getProjectDetail(rs);
     				switch (format) {
     				case "csv":
     					out.write(p.asCSV().getBytes(UTF8));
     					break;
     				case "json":
     					out.write(p.asJson().getBytes(UTF8));
     					break;
+    				}
     				if (++i % gzipFlushSize == 0) {
     					log.debug("flushing output stream");
     					out.flush();
+    				}
+    			}
     			final LocalDateTime end = LocalDateTime.now();
     			if (log.isDebugEnabled()) {
     				log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end));
+    			}
     		} finally {
     			if (out instanceof GZIPOutputStream) {
     				((GZIPOutputStream) out).finish();
+    			}
     			out.close();
+    		}
+    	}
     	private Connection getConn() throws SQLException {
     		final Connection conn = projectApiDataSource.getConnection();
     		conn.setAutoCommit(false);
-...
     		return stm;
+    	}
     	private ProjectDetail getProjectDetail(final ResultSet rs) throws SQLException {
     		return new ProjectDetail()
     				.setProjectId(rs.getString("projectid"))
     				.setAcronym(rs.getString("acronym"))
     				.setCode(rs.getString("code"))
     				.setJsonextrainfo(rs.getString("jsonextrainfo"))
     				.setFundingPath(asList(rs.getArray("fundingpath")));
+    	}
     	private List<String> asList(final Array value) throws SQLException {
     		if (value != null) {
     			final List<Object> list = Arrays.asList((Object[]) value.getArray());

     import java.util.Arrays;
     import java.util.Date;
     import java.util.Map;
     import java.util.concurrent.atomic.AtomicInteger;
     import java.util.zip.GZIPOutputStream;
     import java.util.zip.ZipEntry;
     import java.util.zip.ZipOutputStream;
-...
     import javax.servlet.http.HttpServletRequest;
     import javax.servlet.http.HttpServletResponse;
     import com.google.common.base.Joiner;
     import com.google.common.collect.Maps;
     import com.google.common.xml.XmlEscapers;
     import eu.dnetlib.common.rmi.DNetRestDocumentation;
     import eu.dnetlib.openaire.exporter.datasource.ApiException;
     import eu.dnetlib.openaire.exporter.model.project.ProjectTsv;
     import eu.dnetlib.openaire.exporter.project.repository.ProjectDetailsRepository;
     import eu.dnetlib.openaire.exporter.project.repository.ProjectTsvRepository;
     import org.antlr.stringtemplate.StringTemplate;
     import org.apache.commons.io.IOUtils;
     import org.apache.commons.lang3.StringUtils;
-...
     	@Autowired
     	private ProjectQueryParamsFactory projectQueryParamsFactory;
     	//@Autowired
     	//private ProjectDetailsRepository projectDetailsRepository;
     	@Autowired
     	private ProjectDetailsRepository projectDetailsRepository;
     	@Autowired
     	private ProjectTsvRepository projectTsvRepository;
     	@Deprecated
     	/**
     	 * @deprecated  we should use the codes returned by the yet to come service
-...
     		dao.streamProjectsTSV(st.toString(), out);
+    	}
     	@RequestMapping(value = "/openaire/export/streamProjectDetails.do", method = RequestMethod.GET)
     	void streamProjectDetails(final HttpServletResponse response,
     			@RequestParam(value = "format", required = true) final String format,
     			@RequestParam(value = "compress", required = false) final Boolean compress) throws IOException, SQLException {
     		final String sql = IOUtils.toString(projectDetailsSql.getInputStream(), UTF8);
     	@RequestMapping(value = "/openaire/export/project2tsv2.do", method = RequestMethod.GET)
     	void processTsv2(final HttpServletRequest request, final HttpServletResponse response,
     			@RequestParam(value = "funding", required = true) final String funding,
     			@RequestParam(value = "article293", required = false) final Boolean article293) throws Exception {
     		if (compress != null && compress) {
     			response.setHeader("Content-Encoding", "gzip");
     		final Funding requestedFunding = Funding.valueOf(funding.toUpperCase());
     		final String fundingPrefix = getFundingPrefix(requestedFunding, null);
     		final String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
     		final String filename = "projects_" + funding + "_" + date + ".tsv";
     		response.setContentType("text/tab-separated-values");
     		response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\"");
     		try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) {
     			out.putNextEntry(new ZipEntry(filename));
     			queryForTsv(fundingPrefix, article293).forEach(p -> {
     				try {
     					out.write(Joiner.on('\t').useForNull("").join(p.asList()).getBytes(UTF8));
     					out.write('\n');
     				} catch (IOException e) {
     					throw new RuntimeException(e);
+    				}
     			});
     		} catch (Throwable e) {
     			throw new ApiException(org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error processing the request", e);
+    		}
     		switch (format) {
     		case "csv":
     			response.setContentType("text/csv");
     			break;
     		case "json":
     			response.setContentType("text/plain");
     			break;
     		default: throw new IllegalArgumentException("unsupported format: " + format);
+    	}
     	private Iterable<ProjectTsv> queryForTsv(final String fundingPrefix, final Boolean article293) {
     		log.debug(String.format("fundingPrefix:'%s' and article293:'%s'", fundingPrefix, article293));
     		if (article293 != null) {
     			return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293);
     		} else {
     			return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix);
+    		}
     		final OutputStream outputStream = getOutputStream(response.getOutputStream(), compress);
     		dao.streamProjectDetails(sql, outputStream, format);
+    	}
     	@RequestMapping(value = "/openaire/export/streamProjectDetails2.do", method = RequestMethod.GET)
     	void streamProjectDetails2(final HttpServletResponse response,
     	@RequestMapping(value = "/openaire/export/streamProjectDetails.do", method = RequestMethod.GET)
     	void streamProjectDetails(final HttpServletResponse response,
     			@RequestParam(value = "format", required = true) final String format,
     			@RequestParam(value = "compress", required = false) final Boolean compress) throws IOException, SQLException {
     	/*
     		if (compress != null && compress) {
     			response.setHeader("Content-Encoding", "gzip");
+    		}
     		switch (format) {
     			case "csv":
     				response.setContentType("text/csv");
-...
     							out.flush();
+    						}
     					} catch (IOException e) {
     						log.error("Unable to write to OutputStream", e);
     						throw new RuntimeException(e);
+    					}
+    				}
     			);
-...
+    			}
     			out.close();
+    		}
     	*/
+    	}
     	private OutputStream getOutputStream(final ServletOutputStream outputStream, final Boolean compress) throws IOException {
-...
     	@ExceptionHandler({Exception.class, Throwable.class})
     	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
     	public @ResponseBody ErrorMessage handleSqlException(final Exception e) {
     		log.error(e.getMessage());
     		return new ErrorMessage(e);
     		log.debug(e.getMessage(), e);
     		if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
     			return null;        //socket is closed, cannot return any response
     		} else {
     			return new ErrorMessage(e);
+    		}
+    	}
     	public class ErrorMessage {

modules/dnet-openaire-exporter/trunk/src/main/resources/logback-spring.xml
10	10	<logger name="eu.dnetlib" level="INFO" additivity="false">
11	11	<appender-ref ref="CONSOLE" />
12	12	</logger>
	13	<logger name="eu.dnetlib.openaire.exporter.project.ProjectsController" level="DEBUG" additivity="false">
	14	<appender-ref ref="CONSOLE" />
	15	</logger>
13	16	</configuration>

     spring.datasource.driverClassName=org.postgresql.Driver
     spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect
     spring.jpa.show-sql=false
     spring.jpa.show-sql=true
     spring.jpa.hibernate.ddl-auto=validate
     openaire.exporter.jdbc.url=jdbc:postgresql://localhost:5432/dnet_openaire

Project

General

Profile

D-Net

Revision 48362

Added by Claudio Atzori almost 7 years ago