Project

General

Profile

« Previous | Next » 

Revision 46718

Added by Claudio Atzori about 7 years ago

integrated most recent changes from dnet40

View differences:

modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/clients/DatasourceIndexClient.java
46 46 

  		




  47
  47
  
    
			long count = rsp.getResults().getNumFound();


  		




  48
  48
  
    
			final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument());


  		




  
  49
  
    
			if (log.isDebugEnabled()) {


  		




  
  50
  
    
				log.debug(String.format("got document %s", doc.toString()));


  		




  
  51
  
    
			}


  		




  
  52
  
    
			if (doc.isEmpty()) {


  		




  
  53
  
    
				throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching query: %s", query));


  		




  
  54
  
    
			}


  		




  49
  55
  
    
			final String date = getDate(doc);


  		




  50
  56
  
    

  		




  51
  57
  
    
			return new IndexRecordsInfo(count, date);


  		




  52
  58
  
    
		} catch (final Throwable e) {


  		




  53
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying publications from: " + dsId);


  		




  
  59
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying publications from: " + dsId, e);


  		




  54
  60
  
    
		}


  		




  55
  61
  
    
	}


  		




  56
  62
  
    

  		




  57
  
  
    
	private String getDate(final SolrDocument doc) {


  		




  
  63
  
    
	private String getDate(final SolrDocument doc) throws ApiException {


  		




  58
  64
  
    
		final List<Date> dsversion = (List<Date>) doc.get(DSVERSION);


  		




  
  65
  
    
		if (dsversion == null || dsversion.isEmpty()) {


  		




  
  66
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find %s in matched solr document", DSVERSION));


  		




  
  67
  
    
		}


  		




  59
  68
  
    
		final Date date = Iterables.getLast(dsversion);


  		




  60
  69
  
    

  		




  61
  70
  
    
		return DateFormatUtils.format(date, DatasourceFunctions.DATE_FORMAT);


  		




  ......




  68
  77
  
    
			final String dsversion = doc.get("__dsversion").toString();


  		




  69
  78
  
    
			return StringUtils.substringBefore(dsversion, "T");


  		




  70
  79
  
    
		} catch (CloudIndexClientException e) {


  		




  71
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info);


  		




  
  80
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e);


  		




  72
  81
  
    
		}


  		




  73
  82
  
    
	}


  		




  74
  83
  
    

  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/clients/JdbcDatasourceDao.java
    
  





  47
  47
  
    
				}


  		




  48
  48
  
    
			}


  		




  49
  49
  
    
		} catch(SQLException e) {


  		




  50
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying datasource ids from database");


  		




  
  50
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying datasource ids from database", e);


  		




  51
  51
  
    
		}


  		




  52
  52
  
    
		return res;


  		




  53
  53
  
    
	}


  		




  ......




  64
  64
  
    
				}


  		




  65
  65
  
    
			}


  		




  66
  66
  
    
		} catch(SQLException | IOException e) {


  		




  67
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying datasource '%s' from database", id));


  		




  
  67
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying datasource '%s' from database", id), e);


  		




  68
  68
  
    
		}


  		




  69
  69
  
    
	}


  		




  70
  70
  
    

  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/clients/DatasourceInfoRetriever.java
    
  





  17
  17
  
    
import org.apache.commons.logging.Log;


  		




  18
  18
  
    
import org.apache.commons.logging.LogFactory;


  		




  19
  19
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  
  20
  
    
import org.springframework.beans.factory.annotation.Value;


  		




  20
  21
  
    

  		




  21
  22
  
    
/**


  		




  22
  23
  
    
 * Created by claudio on 20/10/2016.


  		




  ......




  41
  42
  
    

  		




  42
  43
  
    
	private final static int WORKERS = 100;


  		




  43
  44
  
    

  		




  
  45
  
    
	@Value("${datasource.publisher.timeout}")


  		




  
  46
  
    
	private int timeout = 10;


  		




  
  47
  
    

  		




  44
  48
  
    
	public DatasourceInfoRetriever() {


  		




  45
  49
  
    
		service = MoreExecutors.listeningDecorator(


  		




  46
  50
  
    
				new ScheduledThreadPoolExecutor(WORKERS,


  		




  ......




  98
  102
  
    
						innerLatch.countDown();


  		




  99
  103
  
    
					}


  		




  100
  104
  
    
				});


  		




  101
  
  
    
				waitLatch(innerLatch, errors, 10);


  		




  
  105
  
    
				waitLatch(innerLatch, errors, timeout);


  		




  102
  106
  
    

  		




  103
  107
  
    
				outerLatch.countDown();


  		




  104
  108
  
    
			}


  		




  ......




  109
  113
  
    
			}


  		




  110
  114
  
    
		});


  		




  111
  115
  
    

  		




  112
  
  
    
		waitLatch(outerLatch, errors, 10);


  		




  
  116
  
    
		waitLatch(outerLatch, errors, timeout);


  		




  113
  117
  
    

  		




  114
  118
  
    
		if (!errors.isEmpty()) {


  		




  115
  119
  
    
			datasourceResponse.getResponseHeader().setError(Joiner.on("\n").skipNulls().join(Iterables.transform(errors, new Function<Throwable, String>() {


  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/clients/MongoLoggerClient.java
    
  





  11
  11
  
    
import com.mongodb.MongoClient;


  		




  12
  12
  
    
import com.mongodb.client.FindIterable;


  		




  13
  13
  
    
import com.mongodb.client.MongoCollection;


  		




  14
  
  
    
import com.mongodb.client.model.Filters;


  		




  15
  14
  
    
import com.mongodb.client.model.Projections;


  		




  16
  15
  
    
import eu.dnetlib.datasource.publisher.ApiException;


  		




  
  16
  
    
import eu.dnetlib.datasource.publisher.clients.utils.DatasourceFunctions;


  		




  17
  17
  
    
import eu.dnetlib.datasource.publisher.model.*;


  		




  18
  
  
    
import eu.dnetlib.datasource.publisher.clients.utils.DatasourceFunctions;


  		




  19
  18
  
    
import eu.dnetlib.miscutils.datetime.DateUtils;


  		




  20
  19
  
    
import org.apache.commons.lang.StringUtils;


  		




  21
  20
  
    
import org.apache.commons.lang.time.DateFormatUtils;


  		




  ......




  27
  26
  
    
import org.springframework.beans.factory.annotation.Value;


  		




  28
  27
  
    
import org.springframework.cache.annotation.Cacheable;


  		




  29
  28
  
    

  		




  
  29
  
    
import static com.mongodb.client.model.Filters.*;


  		




  
  30
  
    

  		




  
  31
  
    

  		




  
  32
  
    

  		




  30
  33
  
    
/**


  		




  31
  34
  
    
 * Created by claudio on 20/10/2016.


  		




  32
  35
  
    
 */


  		




  ......




  48
  51
  
    
	private static MongoCollection<Document> collection = null;


  		




  49
  52
  
    

  		




  50
  53
  
    
	private Bson getQuery(final String dsId) {


  		




  51
  
  
    
		return Filters.and(dbo("parentDatasourceId", dsId), dbo("system:profileFamily", "aggregator"), dbo("system:isCompletedSuccessfully", "true"));


  		




  
  54
  
    
		return and(


  		




  
  55
  
    
				eq("parentDatasourceId", dsId),


  		




  
  56
  
    
				eq("system:profileFamily", "aggregator"),


  		




  
  57
  
    
				eq("system:isCompletedSuccessfully", "true"));


  		




  52
  58
  
    
	}


  		




  53
  59
  
    

  		




  54
  60
  
    
	private synchronized MongoCollection<Document> getCollection() {


  		




  55
  61
  
    
		if (collection == null) {


  		




  
  62
  
    
			log.info("inizializing mongodb collection ...");


  		




  56
  63
  
    
			collection = datasourcePublisherMongoClient.getDatabase(dbName).getCollection(collectionName);


  		




  57
  64
  
    
		}


  		




  58
  65
  
    
		return collection;


  		




  ......




  128
  135
  
    
	}


  		




  129
  136
  
    

  		




  130
  137
  
    
	private static Bson getFields() {


  		




  131
  
  
    
		return Projections.fields(dbo("system:wfName", 1),


  		




  132
  
  
    
				dbo("system:node:SELECT_MODE:selection", 1),


  		




  133
  
  
    
				dbo("mainlog:sinkSize", 1),


  		




  134
  
  
    
				dbo("mainlog:writeOps", 1),


  		




  135
  
  
    
				dbo("mainlog:total", 1),


  		




  136
  
  
    
				dbo("system:startHumanDate", 1),


  		




  137
  
  
    
				dbo("system:profileName", 1));


  		




  
  138
  
    
		return Projections.fields(


  		




  
  139
  
    
				eq("system:wfName", 1),


  		




  
  140
  
    
				eq("system:node:SELECT_MODE:selection", 1),


  		




  
  141
  
    
				eq("mainlog:sinkSize", 1),


  		




  
  142
  
    
				eq("mainlog:writeOps", 1),


  		




  
  143
  
    
				eq("mainlog:total", 1),


  		




  
  144
  
    
				eq("system:startHumanDate", 1),


  		




  
  145
  
    
				eq("system:profileName", 1));


  		




  138
  146
  
    
	}


  		




  139
  147
  
    

  		




  140
  148
  
    
	private static BasicDBObject dbo(final String key, final Object value) {


  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/ApiException.java
    
  





  8
  8
  
    
		super(msg);


  		




  9
  9
  
    
		this.code = code;


  		




  10
  10
  
    
	}


  		




  
  11
  
    

  		




  
  12
  
    
	public ApiException (int code, String msg, Throwable e) {


  		




  
  13
  
    
		super(msg, e);


  		




  
  14
  
    
		this.code = code;


  		




  
  15
  
    
	}


  		




  11
  16
  
    
}


  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/java/eu/dnetlib/datasource/publisher/DatasourcesApiController.java
    
  





  1
  1
  
    
package eu.dnetlib.datasource.publisher;


  		




  2
  2
  
    

  		




  3
  
  
    
import java.util.List;


  		




  4
  
  
    

  		




  5
  
  
    
import com.google.common.collect.Lists;


  		




  6
  3
  
    
import eu.dnetlib.common.rmi.DNetRestDocumentation;


  		




  7
  4
  
    
import eu.dnetlib.datasource.publisher.clients.ClientResponse;


  		




  8
  5
  
    
import eu.dnetlib.datasource.publisher.clients.DatasourceInfoRetriever;


  		




  9
  6
  
    
import eu.dnetlib.datasource.publisher.model.DatasourceResponse;


  		




  10
  
  
    

  		




  11
  7
  
    
import eu.dnetlib.datasource.publisher.model.IdentifiersResponse;


  		




  12
  8
  
    
import eu.dnetlib.datasource.publisher.model.Response;


  		




  13
  
  
    
import io.swagger.annotations.*;


  		




  14
  
  
    

  		




  
  9
  
    
import io.swagger.annotations.ApiParam;


  		




  15
  10
  
    
import org.apache.commons.logging.Log;


  		




  16
  11
  
    
import org.apache.commons.logging.LogFactory;


  		




  17
  12
  
    
import org.springframework.beans.BeanUtils;


  		












  

    
      modules/dnet-datasource-rest-api/trunk/src/main/resources/eu/dnetlib/datasource/publisher/applicationContext-dnet-datasource-publisher.properties
    
  





  14
  14
  
    
datasource.publisher.jdbc.minIdle   =   1


  		




  15
  15
  
    
datasource.publisher.jdbc.maxIdle   =   20


  		




  16
  16
  
    
datasource.publisher.getsingle.sql  =   /eu/dnetlib/datasource/publisher/getDatasources.sql.st


  		




  
  17
  
    
datasource.publisher.timeout        =   10


  		












Also available in:  Unified diff