Project

General

Profile

« Previous | Next » 

Revision 46696

Added by Claudio Atzori over 7 years ago

removed used of deprecated method

View differences:

modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/mongo/MongoDumpPlugin.java
3 3 
import java.io.File;
4 4 
import java.io.FileNotFoundException;
5 5 
import java.io.FileReader;
6 
import java.util.Iterator;
6 
import java.io.IOException;
7 7 
import java.util.stream.Stream;
8 8 

  		




  9
  
  
    
import org.springframework.stereotype.Component;


  		




  10
  
  
    

  		




  11
  9
  
    
import eu.dnetlib.miscutils.streams.DnetStreamSupport;


  		




  
  10
  
    
import eu.dnetlib.msro.exceptions.CollectorServiceRuntimeException;


  		




  12
  11
  
    
import eu.dnetlib.msro.workers.aggregation.collect.CollectException;


  		




  13
  12
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;


  		




  14
  13
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;


  		




  15
  14
  
    
import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  15
  
    
import org.springframework.stereotype.Component;


  		




  16
  16
  
    

  		




  17
  17
  
    
@Component


  		




  18
  18
  
    
@DnetCollectorPlugin("mongoDump")


  		




  ......




  29
  29
  
    
			if (f.exists() == false) { throw new CollectException("the file at url " + baseUrl + " does not exists"); }


  		




  30
  30
  
    

  		




  31
  31
  
    
			final FileReader reader = new FileReader(f);


  		




  32
  
  
    

  		




  33
  
  
    
			final Iterator<String> iter = new MongoDumpIterator(reader);


  		




  34
  
  
    

  		




  35
  
  
    
			return DnetStreamSupport.generateStreamFromIterator(iter);


  		




  
  32
  
    
			return DnetStreamSupport.stream(new MongoDumpIterator(reader)).onClose(() -> {


  		




  
  33
  
    
				try {


  		




  
  34
  
    
					reader.close();


  		




  
  35
  
    
				} catch (IOException e) {


  		




  
  36
  
    
					throw new CollectorServiceRuntimeException(e);


  		




  
  37
  
    
				}


  		




  
  38
  
    
			});


  		




  36
  39
  
    
		} catch (final FileNotFoundException e) {


  		




  37
  40
  
    
			throw new CollectException("Error unable to open inputStream", e);


  		




  38
  41
  
    
		}


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/archive/zip/ZipCollectorPlugin.java
    
  





  3
  3
  
    
import java.io.File;


  		




  4
  4
  
    
import java.net.MalformedURLException;


  		




  5
  5
  
    
import java.net.URL;


  		




  6
  
  
    
import java.util.Iterator;


  		




  7
  6
  
    
import java.util.stream.Stream;


  		




  8
  7
  
    

  		




  9
  
  
    
import org.springframework.stereotype.Component;


  		




  10
  
  
    

  		




  11
  8
  
    
import eu.dnetlib.miscutils.streams.DnetStreamSupport;


  		




  12
  9
  
    
import eu.dnetlib.msro.workers.aggregation.collect.CollectException;


  		




  13
  10
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;


  		




  14
  11
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;


  		




  15
  12
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.oai.engine.XmlCleaner;


  		




  16
  
  
    
// import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  17
  13
  
    
import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  14
  
    
import org.springframework.stereotype.Component;


  		




  18
  15
  
    

  		




  
  16
  
    
// import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  17
  
    

  		




  19
  18
  
    
/**


  		




  20
  19
  
    
 * Collector pluging for collecting a zipped folder of records


  		




  21
  20
  
    
 *


  		




  ......




  38
  37
  
    
			final File zipFile = new File(zipUrl.getPath());


  		




  39
  38
  
    
			if (!zipFile.exists()) { throw new CollectException(String.format("The base ULR %s, does not exist", zipFile.getPath())); }


  		




  40
  39
  
    

  		




  41
  
  
    
			final Iterator<String> zipIterator = new ZipIterator(zipFile.getAbsolutePath());


  		




  42
  
  
    

  		




  43
  
  
    
			return DnetStreamSupport.generateStreamFromIterator(zipIterator)


  		




  
  40
  
    
			return DnetStreamSupport.stream(new ZipIterator(zipFile.getAbsolutePath()))


  		




  44
  41
  
    
					.map(s -> s.startsWith("\uFEFF") ? s.substring(1) : s)


  		




  45
  42
  
    
					.map(XmlCleaner::cleanAllEntities);


  		




  46
  43
  
    
		} catch (final MalformedURLException e) {


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/archive/targz/TarGzCollectorPlugin.java
    
  





  40
  40
  
    

  		




  41
  41
  
    
			final TarGzIterator tgzIterator = new TarGzIterator(tarGzFile.getAbsolutePath());


  		




  42
  42
  
    

  		




  43
  
  
    
			return DnetStreamSupport.generateStreamFromIterator(tgzIterator)


  		




  
  43
  
    
			return DnetStreamSupport.stream(tgzIterator)


  		




  44
  44
  
    
					.map(s -> s.startsWith("\uFEFF") ? s.substring(1) : s)


  		




  45
  45
  
    
					.map(XmlCleaner::cleanAllEntities);


  		




  46
  46
  
    
		} catch (final MalformedURLException e) {


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/filesystem/FilesystemCollectorPlugin.java
    
  





  44
  44
  
    
			if (!baseDir.exists()) { throw new CollectException(String.format("The base ULR %s, does not exist", basePath.getPath())); }


  		




  45
  45
  
    
			final String extension = interfaceDescriptor.getParams().get("extensions");


  		




  46
  46
  
    

  		




  47
  
  
    
			final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), extension);


  		




  48
  
  
    

  		




  49
  
  
    
			return DnetStreamSupport.generateStreamFromIterator(fsi)


  		




  
  47
  
    
			return DnetStreamSupport.stream(new FileSystemIterator(baseDir.getAbsolutePath(), extension))


  		




  50
  48
  
    
					.map(inputFileName -> {


  		




  51
  49
  
    
						try (FileInputStream fileInputStream = new FileInputStream(inputFileName)) {


  		




  52
  50
  
    
							final String s = IOUtils.toString(fileInputStream);


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/oaisets/OaiSetsCollectorPlugin.java
    
  





  26
  26
  
    

  		




  27
  27
  
    
		if ((baseUrl == null) || baseUrl.isEmpty()) { throw new CollectException("Param 'baseurl' is null or empty"); }


  		




  28
  28
  
    

  		




  29
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(new OaiSetsIterator(baseUrl, httpConnector));


  		




  
  29
  
    
		return DnetStreamSupport.stream(new OaiSetsIterator(baseUrl, httpConnector));


  		




  30
  30
  
    
	}


  		




  31
  31
  
    

  		




  32
  32
  
    
}


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/oai/OaiCollectorPlugin.java
    
  





  62
  62
  
    
				.map(set -> new OaiIterator(baseUrl, mdFormat, set, fromDate, untilDate, httpConnector))


  		




  63
  63
  
    
				.iterator();


  		




  64
  64
  
    

  		




  65
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(Iterators.concat(iterator));


  		




  
  65
  
    
		return DnetStreamSupport.stream(Iterators.concat(iterator));


  		




  66
  66
  
    

  		




  67
  67
  
    
	}


  		




  68
  68
  
    

  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/ftp/FtpCollectorPlugin.java
    
  





  1
  1
  
    
package eu.dnetlib.msro.workers.aggregation.collect.plugins.ftp;


  		




  2
  2
  
    

  		




  3
  
  
    
import java.util.Iterator;


  		




  4
  3
  
    
import java.util.Set;


  		




  5
  4
  
    
import java.util.stream.Stream;


  		




  6
  5
  
    

  		




  7
  
  
    
import org.springframework.stereotype.Component;


  		




  8
  
  
    

  		




  9
  6
  
    
import com.google.common.base.Splitter;


  		




  10
  7
  
    
import com.google.common.collect.Sets;


  		




  11
  
  
    

  		




  12
  8
  
    
import eu.dnetlib.miscutils.streams.DnetStreamSupport;


  		




  13
  9
  
    
import eu.dnetlib.msro.workers.aggregation.collect.CollectException;


  		




  14
  10
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;


  		




  ......




  16
  12
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;


  		




  17
  13
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.ProtocolParameterType;


  		




  18
  14
  
    
import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  15
  
    
import org.springframework.stereotype.Component;


  		




  19
  16
  
    

  		




  20
  17
  
    
/**


  		




  21
  18
  
    
 * @author Author: Andrea Mannocci


  		




  ......




  49
  46
  
    

  		




  50
  47
  
    
		final Set<String> extensionsSet = parseSet(extensions);


  		




  51
  48
  
    

  		




  52
  
  
    
		final Iterator<String> iter = new FtpIterator(baseUrl, username, password, isRecursive, extensionsSet);


  		




  53
  
  
    

  		




  54
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(iter);


  		




  
  49
  
    
		return DnetStreamSupport.stream(new FtpIterator(baseUrl, username, password, isRecursive, extensionsSet));


  		




  55
  50
  
    
	}


  		




  56
  51
  
    

  		




  57
  52
  
    
	private Set<String> parseSet(final String extensions) {


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/csv/HttpCSVCollectorPlugin.java
    
  





  47
  47
  
    
	 */


  		




  48
  48
  
    
	@Override


  		




  49
  49
  
    
	public Stream<String> collect(final InterfaceDescriptor descriptor, final String fromDate, final String untilDate) throws CollectException {


  		




  50
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(getIterator(descriptor));


  		




  
  50
  
    
		return DnetStreamSupport.stream(getIterator(descriptor));


  		




  51
  51
  
    
	}


  		




  52
  52
  
    

  		




  53
  53
  
    
	private Iterator<String> getIterator(final InterfaceDescriptor descriptor) {


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/httplist/HttpListCollectorPlugin.java
    
  





  1
  1
  
    
package eu.dnetlib.msro.workers.aggregation.collect.plugins.httplist;


  		




  2
  2
  
    

  		




  3
  
  
    
import java.util.Iterator;


  		




  4
  3
  
    
import java.util.stream.Stream;


  		




  5
  4
  
    

  		




  6
  
  
    
import org.springframework.stereotype.Component;


  		




  7
  
  
    

  		




  8
  5
  
    
import eu.dnetlib.miscutils.streams.DnetStreamSupport;


  		




  9
  6
  
    
import eu.dnetlib.msro.workers.aggregation.collect.CollectException;


  		




  10
  7
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;


  		




  11
  8
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorParam;


  		




  12
  9
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;


  		




  13
  10
  
    
import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  11
  
    
import org.springframework.stereotype.Component;


  		




  14
  12
  
    

  		




  15
  13
  
    
@Component


  		




  16
  14
  
    
@DnetCollectorPlugin(value = "httpList", parameters = {


  		




  ......




  24
  22
  
    
		final String baseUrl = interfaceDescriptor.getBaseUrl();


  		




  25
  23
  
    
		final String listAddress = interfaceDescriptor.getParams().get("listUrl");


  		




  26
  24
  
    

  		




  27
  
  
    
		final Iterator<String> iter = new HttpListIterator(baseUrl, listAddress);


  		




  28
  
  
    

  		




  29
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(iter);


  		




  
  25
  
    
		return DnetStreamSupport.stream(new HttpListIterator(baseUrl, listAddress));


  		




  30
  26
  
    
	}


  		




  31
  27
  
    
}


  		












  

    
      modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workers/aggregation/collect/plugins/sftp/SftpCollectorPlugin.java
    
  





  1
  1
  
    
package eu.dnetlib.msro.workers.aggregation.collect.plugins.sftp;


  		




  2
  2
  
    

  		




  3
  
  
    
import java.util.Iterator;


  		




  4
  3
  
    
import java.util.Set;


  		




  5
  4
  
    
import java.util.stream.Stream;


  		




  6
  5
  
    

  		




  7
  
  
    
import org.springframework.stereotype.Component;


  		




  8
  
  
    

  		




  9
  6
  
    
import com.google.common.base.Splitter;


  		




  10
  7
  
    
import com.google.common.collect.Sets;


  		




  11
  
  
    

  		




  12
  8
  
    
import eu.dnetlib.miscutils.streams.DnetStreamSupport;


  		




  13
  9
  
    
import eu.dnetlib.msro.workers.aggregation.collect.CollectException;


  		




  14
  10
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.CollectorPlugin;


  		




  ......




  16
  12
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.DnetCollectorPlugin;


  		




  17
  13
  
    
import eu.dnetlib.msro.workers.aggregation.collect.plugins.ProtocolParameterType;


  		




  18
  14
  
    
import eu.dnetlib.msro.workers.aggregation.objects.InterfaceDescriptor;


  		




  
  15
  
    
import org.springframework.stereotype.Component;


  		




  19
  16
  
    

  		




  20
  17
  
    
/**


  		




  21
  18
  
    
 * Created by andrea on 11/01/16.


  		




  ......




  53
  50
  
    

  		




  54
  51
  
    
		final Set<String> extensionsSet = parseSet(extensions);


  		




  55
  52
  
    

  		




  56
  
  
    
		final Iterator<String> iter = getSftpIteratorFactory().newIterator(baseUrl, username, password, isRecursive, extensionsSet, fromDate);


  		




  57
  
  
    

  		




  58
  
  
    
		return DnetStreamSupport.generateStreamFromIterator(iter);


  		




  
  53
  
    
		return DnetStreamSupport.stream(getSftpIteratorFactory().newIterator(baseUrl, username, password, isRecursive, extensionsSet, fromDate));


  		




  59
  54
  
    
	}


  		




  60
  55
  
    

  		




  61
  56
  
    
	private Set<String> parseSet(final String extensions) {


  		












Also available in:  Unified diff