Revision 53677
Added by Giorgos Papanikos about 6 years ago
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/schemaorg/httpapi/kaggle/KaggleRepositoryIterable.java | ||
|---|---|---|
| 115 | 115 |
} |
| 116 | 116 |
|
| 117 | 117 |
public void bootstrap() {
|
| 118 |
this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize(), true);
|
|
| 118 |
this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize()); |
|
| 119 | 119 |
|
| 120 | 120 |
ExecutorService executor = Executors.newSingleThreadExecutor(); |
| 121 | 121 |
executor.execute(new Harvester()); |
| ... | ... | |
| 157 | 157 |
if (urlFragment == null || urlFragment.trim().length() == 0) continue; |
| 158 | 158 |
String endpoint = String.format("%s%s", options.getResponseBaseDatasetUrl(), urlFragment);
|
| 159 | 159 |
|
| 160 |
log.debug("adding endpoint in queue");
|
|
| 161 |
log.debug("queue size: " + queue.size());
|
|
| 160 | 162 |
queue.put(endpoint); |
| 163 |
log.debug("endpoint added in queue");
|
|
| 164 |
log.debug("queue size: " + queue.size());
|
|
| 161 | 165 |
} |
| 162 | 166 |
|
| 163 | 167 |
if (readDatasets >= totalDatasets) break; |
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/schemaorg/sitemapindex/SitemapIndexRepositoryIterable.java | ||
|---|---|---|
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 | 64 |
public void bootstrap() {
|
| 65 |
this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize(), true);
|
|
| 65 |
this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize()); |
|
| 66 | 66 |
|
| 67 | 67 |
ExecutorService executor = Executors.newSingleThreadExecutor(); |
| 68 | 68 |
executor.execute(new Harvester()); |
| ... | ... | |
| 99 | 99 |
String endpoint = sitemapFileIterator.next(); |
| 100 | 100 |
if(endpoint == null) continue;; |
| 101 | 101 |
|
| 102 |
log.debug("adding endpoint in queue");
|
|
| 103 |
log.debug("queue size: " + queue.size());
|
|
| 102 | 104 |
queue.put(endpoint); |
| 105 |
log.debug("endpoint added in queue");
|
|
| 106 |
log.debug("queue size: " + queue.size());
|
|
| 103 | 107 |
} |
| 104 | 108 |
} |
| 105 | 109 |
}catch(Exception ex){
|
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/schemaorg/RepositoryQueueIterator.java | ||
|---|---|---|
| 71 | 71 |
} |
| 72 | 72 |
|
| 73 | 73 |
private String poll(){
|
| 74 |
String item = null; |
|
| 75 |
log.debug("retrieving endpoint from queue");
|
|
| 76 |
log.debug("queue size: " + queue.size());
|
|
| 74 | 77 |
if(this.options.getBlockPolling()) {
|
| 75 | 78 |
try {
|
| 76 |
return this.queue.poll(this.options.getPollTimeout(), this.options.getPollTimeoutUnit());
|
|
| 79 |
item = this.queue.poll(this.options.getPollTimeout(), this.options.getPollTimeoutUnit());
|
|
| 77 | 80 |
} catch (InterruptedException ex) {
|
| 78 | 81 |
log.warn(String.format("could not poll elements from queue for more than %s %s. throwing", this.options.getPollTimeout(), this.options.getPollTimeoutUnit()));
|
| 79 | 82 |
throw new NoSuchElementException(ex.getMessage()); |
| 80 | 83 |
} |
| 81 | 84 |
} |
| 82 | 85 |
else {
|
| 83 |
return this.queue.poll();
|
|
| 86 |
item = this.queue.poll();
|
|
| 84 | 87 |
} |
| 88 |
log.debug("retrieved endpoint from queue");
|
|
| 89 |
log.debug("queue size: " + queue.size());
|
|
| 90 |
return item; |
|
| 85 | 91 |
} |
| 86 | 92 |
} |
Also available in: Unified diff
added more logging and removed fair option from blocking queue