Revision 33870
Added by Andrea Mannocci over 9 years ago
| FeedSpeedTest.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mdstore.modular.mongodb; |
| 2 | 2 |
|
| 3 |
import java.io.File; |
|
| 4 |
import java.io.FileInputStream; |
|
| 3 | 5 |
import java.io.IOException; |
| 4 | 6 |
import java.util.Iterator; |
| 5 | 7 |
import java.util.UUID; |
| 6 | 8 |
|
| 7 | 9 |
import org.apache.commons.io.IOUtils; |
| 8 | 10 |
import org.junit.Ignore; |
| 11 |
import org.junit.Test; |
|
| 9 | 12 |
import org.junit.runner.RunWith; |
| 10 | 13 |
import org.springframework.beans.factory.annotation.Autowired; |
| 11 | 14 |
import org.springframework.core.io.ClassPathResource; |
| ... | ... | |
| 24 | 27 |
@ContextConfiguration(classes = ConfigurationTestConfig.class) |
| 25 | 28 |
public class FeedSpeedTest {
|
| 26 | 29 |
|
| 27 |
private static final int N_RECORDS = 3000000;
|
|
| 30 |
private static final int N_RECORDS = 68271;
|
|
| 28 | 31 |
|
| 29 | 32 |
@Autowired |
| 30 | 33 |
private DB db; |
| ... | ... | |
| 39 | 42 |
private RecordParser recordParser; |
| 40 | 43 |
|
| 41 | 44 |
@Ignore |
| 42 |
public void testFeedSpeed() throws MDStoreServiceException, IOException {
|
|
| 45 |
@Test |
|
| 46 |
public void testSpeedFromFolder() throws IOException {
|
|
| 47 |
Iterable<String> iterable = new Iterable<String>() {
|
|
| 48 |
|
|
| 49 |
private int counter = 0; |
|
| 50 |
private double last = System.currentTimeMillis(); |
|
| 51 |
|
|
| 52 |
@Override |
|
| 53 |
public Iterator<String> iterator() {
|
|
| 54 |
return new Iterator<String>() {
|
|
| 55 |
|
|
| 56 |
@Override |
|
| 57 |
public boolean hasNext() {
|
|
| 58 |
return counter < N_RECORDS; |
|
| 59 |
} |
|
| 60 |
|
|
| 61 |
@Override |
|
| 62 |
public String next() {
|
|
| 63 |
if (counter % 10000 == 0) {
|
|
| 64 |
System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
|
|
| 65 |
last = System.currentTimeMillis(); |
|
| 66 |
} |
|
| 67 |
|
|
| 68 |
File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
|
|
| 69 |
if (f.exists()) {
|
|
| 70 |
try {
|
|
| 71 |
FileInputStream fileInputStream = new FileInputStream(f); |
|
| 72 |
String s = IOUtils.toString(fileInputStream); |
|
| 73 |
fileInputStream.close(); |
|
| 74 |
return s; |
|
| 75 |
} catch (Exception e) {
|
|
| 76 |
return null; |
|
| 77 |
} |
|
| 78 |
} else {
|
|
| 79 |
try {
|
|
| 80 |
FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));
|
|
| 81 |
String s = IOUtils.toString(fileInputStream); |
|
| 82 |
fileInputStream.close(); |
|
| 83 |
return s; |
|
| 84 |
} catch (Exception e) {
|
|
| 85 |
return null; |
|
| 86 |
} |
|
| 87 |
} |
|
| 88 |
} |
|
| 89 |
|
|
| 90 |
@Override |
|
| 91 |
public void remove() {}
|
|
| 92 |
}; |
|
| 93 |
} |
|
| 94 |
}; |
|
| 95 |
|
|
| 96 |
MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test"), recordParser, true);
|
|
| 97 |
mdStore.feed(iterable, false); |
|
| 98 |
} |
|
| 99 |
|
|
| 100 |
@Ignore |
|
| 101 |
@Test |
|
| 102 |
public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {
|
|
| 43 | 103 |
MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test"), recordParser, false);
|
| 44 |
|
|
| 45 | 104 |
mdStore.feed(new Iterable<String>() {
|
| 46 | 105 |
|
| 47 | 106 |
private int counter = 0; |
| ... | ... | |
| 64 | 123 |
System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
|
| 65 | 124 |
last = System.currentTimeMillis(); |
| 66 | 125 |
} |
| 67 |
return templateRecord.replace("{id}", String.valueOf(counter++));
|
|
| 126 |
|
|
| 127 |
File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
|
|
| 128 |
if (f.exists()) {
|
|
| 129 |
try {
|
|
| 130 |
FileInputStream fileInputStream = new FileInputStream(f); |
|
| 131 |
String s = IOUtils.toString(fileInputStream); |
|
| 132 |
fileInputStream.close(); |
|
| 133 |
return s; |
|
| 134 |
} catch (Exception e) {
|
|
| 135 |
return null; |
|
| 136 |
} |
|
| 137 |
} else {
|
|
| 138 |
counter++; |
|
| 139 |
try {
|
|
| 140 |
FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));
|
|
| 141 |
String s = IOUtils.toString(fileInputStream); |
|
| 142 |
fileInputStream.close(); |
|
| 143 |
return s; |
|
| 144 |
} catch (Exception e) {
|
|
| 145 |
return null; |
|
| 146 |
} |
|
| 147 |
} |
|
| 68 | 148 |
} |
| 69 | 149 |
|
| 70 | 150 |
@Override |
| ... | ... | |
| 73 | 153 |
} |
| 74 | 154 |
}, false); |
| 75 | 155 |
} |
| 76 |
|
|
| 77 | 156 |
} |
Also available in: Unified diff