D-Net

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.util.Iterator;

import java.util.UUID;

import org.apache.commons.io.IOUtils;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.core.io.ClassPathResource;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import com.mongodb.DB;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.RecordParser;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class FeedSpeedTest {

	private static final int N_RECORDS = 68271;

	@Autowired

	private DB db;

	@Autowired

	private MDStoreTransactionManager manager;

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private RecordParser recordParser;

	@Ignore

	@Test

	public void testSpeedFromFolder() throws IOException {

		Iterable<String> iterable = new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {}

				};

			}

		};

		MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test"), recordParser, true);

		mdStore.feed(iterable, false);

	}

	@Ignore

	@Test

	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {

		MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test"), recordParser, false);

		mdStore.feed(new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")

					.getInputStream());

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							counter++;

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {}

				};

			}

		}, false);

	}

}