Project

General

Profile

« Previous | Next » 

Revision 58480

Added by Jochen Schirrwagen about 4 years ago

added user-agent string as it is required by some servers to accept requests

View differences:

modules/dnet-download-service/trunk/src/test/java/DownloadWorkerTest.java
51 51 
	@Ignore
52 52 
	@Test
53 53 
	public void testSSL(){
54 
		String url = "https://econstor.eu/bitstream/10419/45606/1/658945068.pdf";
55 
		dw.doDownload(1, reportMap, getDownloadItem("658945068.pdf", "658945068", url));
54 
		// String url = "https://econstor.eu/bitstream/10419/45606/1/658945068.pdf";
55 
		String url = "https://www.preprints.org/manuscript/201909.0029/v1/download";
56 
		dw.doDownload(1, reportMap, getDownloadItem("download.pdf", "download", url));
56 57 

  		




  57
  58
  
    
		System.out.println("Report map status "+reportMap.getStatus());


  		




  58
  59
  
    
		assertEquals(1,reportMap.getTotalDownloaded());


  		












  

    
      modules/dnet-download-service/trunk/src/main/java/eu/dnetlib/data/download/worker/DownloadWorker.java
    
  





  63
  63
  
    
	 * ms to wait between two subsequent download request


  		




  64
  64
  
    
	 **/


  		




  65
  65
  
    
	private int sleepTimeMs;


  		




  
  66
  
    
	


  		




  
  67
  
    
	private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";


  		




  66
  68
  
    

  		




  67
  69
  
    
	/**


  		




  68
  70
  
    
	 * Instantiates a new download worker.


  		




  ......




  163
  165
  
    

  		




  164
  166
  
    
			log.debug(threadId + ": opening connection " + toDownload);


  		




  165
  167
  
    
			final URLConnection connection = toDownload.openConnection();


  		




  
  168
  
    
			// set user-agent


  		




  
  169
  
    
			connection.setRequestProperty("User-Agent", userAgent);


  		




  166
  170
  
    
			connection.setConnectTimeout(getConnectTimeoutMs());


  		




  167
  171
  
    
			connection.setReadTimeout(getReadTimeoutMs());


  		




  168
  172
  
    
			connection.setRequestProperty("User-Agent", "OpenAIRE aggregation system/2.2.5 ; (OpenAccess aggregator; mailto:helpdesk@openaire.eu) ");


  		




  ......




  190
  194
  
    
		conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.


  		




  191
  195
  
    
		conn.setReadTimeout(getReadTimeoutMs());


  		




  192
  196
  
    
		conn.setConnectTimeout(getConnectTimeoutMs());


  		




  
  197
  
    
		// set user-agent


  		




  
  198
  
    
		conn.setRequestProperty("User-Agent", userAgent);


  		




  
  199
  
    

  		




  193
  200
  
    
		String location = inputURL.toString();


  		




  194
  201
  
    
		log.debug(threadId + " : followURL connecting  " + inputURL);


  		




  195
  202
  
    
		conn.connect();


  		












Also available in:  Unified diff