Revision 52230
Added by Miriam Baglioni almost 6 years ago
| HTTPWithFileNameCollectorIterable.java | ||
|---|---|---|
| 107 | 107 |
Connector c = new Connector(); |
| 108 | 108 |
String url; |
| 109 | 109 |
while((metas.size()>0 || urls.size() > 0 ) && queue.size()<100){
|
| 110 |
log.debug("metas.size() = " + metas.size() + " urls.size() = " + urls.size() + " queue.size() = " +queue.size());
|
|
| 110 | 111 |
if (metas.size() > 0){
|
| 111 | 112 |
url = metas.remove(0); |
| 112 | 113 |
try {
|
| 113 | 114 |
c.get(url); |
| 114 | 115 |
} catch (CollectorServiceException e) {
|
| 115 |
log.error("Impossible to collect url: " + url + " error: " + e.getMessage());
|
|
| 116 |
log.info("Impossible to collect url: " + url + " error: " + e.getMessage());
|
|
| 116 | 117 |
} |
| 117 | 118 |
if(c.isStatusOk()){
|
| 118 | 119 |
try {
|
| ... | ... | |
| 125 | 126 |
total++; |
| 126 | 127 |
} |
| 127 | 128 |
} catch (InterruptedException e) {
|
| 128 |
log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );
|
|
| 129 |
log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );
|
|
| 129 | 130 |
|
| 131 |
|
|
| 130 | 132 |
} |
| 131 | 133 |
} |
| 132 | 134 |
}else{
|
| ... | ... | |
| 134 | 136 |
try {
|
| 135 | 137 |
c.get(url); |
| 136 | 138 |
} catch (CollectorServiceException e) {
|
| 137 |
log.error("Impossible to collect url: " + url + " error: " + e.getMessage());
|
|
| 139 |
log.info("Impossible to collect url: " + url + " error: " + e.getMessage());
|
|
| 138 | 140 |
} |
| 139 | 141 |
if(c.isStatusOk()){
|
| 140 | 142 |
if (c.responseTypeContains("text/html")){
|
| ... | ... | |
| 144 | 146 |
try {
|
| 145 | 147 |
queue.put(addFilePath(c.getResponse(),url, c.responseTypeContains("application/json")));
|
| 146 | 148 |
} catch (InterruptedException e) {
|
| 147 |
log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );
|
|
| 149 |
log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );
|
|
| 148 | 150 |
} |
| 149 | 151 |
} |
| 150 | 152 |
} |
Also available in: Unified diff
some logs for debugging reasons added