D-Net

package eu.dnetlib.data.mdstore.modular.newmongodb;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoCollection;

import eu.dnetlib.data.mdstore.modular.mongodb.MDRecordParser;

import eu.dnetlib.data.mdstore.modular.newmongodb.model.ConfigurationMongo;

import eu.dnetlib.data.mdstore.modular.newmongodb.model.MDStoreEncoder;

import eu.dnetlib.data.mdstore.modular.newmongodb.model.MDStoreRecord;

import eu.dnetlib.data.mdstore.modular.newmongodb.model.MongoClientConfiguration;

import eu.dnetlib.data.mdstore.modular.newmongodb.repository.MetadataRepository;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.enabling.tools.DnetStreamSupport;

import eu.dnetlib.rmi.data.DocumentNotFoundException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.bson.Document;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import java.io.IOException;

import java.util.*;

/**

 * Created by sandro on 5/23/17.

 */

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = {MongoClientConfiguration.class, ConfigurationMongo.class})

public class MongoMDStoreTest {

    @Autowired

    private MongoClient client;

    @Autowired

    private MetadataRepository metadataRepository;

    private MongoCollection<Document> metadata;

    private MongoCollection<Document> collection;

    private MongoCollection<Document> discardedCollection;

    @Before

    public void initializeCollections() {

        metadata = client.getDatabase("mdstore").getCollection("metadata");

        collection = client.getDatabase("mdstore").getCollection("collection");

        discardedCollection = client.getDatabase("mdstore").getCollection("discarded");

    }

    @Test

    public void testIterator() throws IOException {

        int i = 0;

        for (String s : MongoTestUtility.createLongIterable(100)) {

            i++;

        }

        Assert.assertEquals(100, i);

    }

    @Test

    public void testSpeedParsing() throws IOException {

        final Map<String, String> mdFormat = new HashMap<>();

        mdFormat.put("subject", "//*[local-name()='subject']");

        mdFormat.put("date", "//*[local-name()='date']");

        final long start = System.currentTimeMillis();

        long count = DnetStreamSupport.generateParallelStreamFromIterator(MongoTestUtility.createLongIterable(10000).iterator()).

                map(it ->

                        MDStoreRecord.generateRecord(it, new MDRecordParser(mdFormat))

                ).count();

        final long total = System.currentTimeMillis() - start;

        System.out.println("count = " + count + " in  " + total);

    }

    private void feedMDStore(final int numberOfItems, final MongoMDStore mdStore) throws Exception {

        long start = System.currentTimeMillis();

        mdStore.feed(MongoTestUtility.createLongIterable(numberOfItems));

        long end = System.currentTimeMillis();

        System.out.println("DONE  100 in " + (end - start) + " ms and size " + collection.count());

    }

    @Test

    public void testFeedRecord() throws Exception {

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(100, store);

        MDStoreEncoder.decodeAllMDStoreRecords(collection.find()).forEach(

                record ->

                {

                    Assert.assertNotNull(record.getBody());

                    Assert.assertNotNull(record.getExtractedValues());

                    Assert.assertNotNull(record.getExtractedValues().get("subject"));

                    Assert.assertTrue(record.getExtractedValues().get("subject").size() > 0);

                    Assert.assertNotNull(record.getExtractedValues().get("date"));

                    Assert.assertTrue(record.getExtractedValues().get("date").size() > 0);

                }

        );

    }

    @Test

    public void isIndexedTest() throws Exception {

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(3, store);

        Assert.assertTrue(store.isIndexed());

    }

    @Test

    public void testReadWithResultSet() throws Exception {

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(3, store);

        Map<String, String> query = new HashMap<>();

        query.put("date", "1978");

        query.put("subject", "Sea");

        metadataRepository.bootstrap();

        ResultSetListener<String> deliver = store.deliver(null, null, query);

        Assert.assertTrue(deliver.getCount()==0);

        Assert.assertTrue(deliver.getTotal()>0);

        Assert.assertTrue(deliver.hasNext());

        while (deliver.hasNext()) {

            final String nextRecord = deliver.next();

            Assert.assertNotNull(nextRecord);

        }

        deliver = store.deliver(null, "" + System.currentTimeMillis(), null);

        Assert.assertTrue(deliver.hasNext());

        int i = 0;

        while (deliver.hasNext()) {

            final String nextRecord = deliver.next();

            Assert.assertNotNull(nextRecord);

            i ++;

        }

        Assert.assertEquals(2, i);

        deliver = store.deliver( "" + System.currentTimeMillis(),null, null);

        Assert.assertFalse(deliver.hasNext());

    }

    @Test(expected = DocumentNotFoundException.class)

    public void getAndDeleteRecordTest() throws Exception {

        final String id = "r3d100010134::0";

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(10, store);

        String record = store.getRecord(id);

        Assert.assertFalse(StringUtils.isEmpty(record));

        store.deleteRecord(id);

        //NOW It Should Run an exception

        store.getRecord(id);

    }

    @Test

    public void testReadIterable() throws Exception {

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(300, store);

        Map<String, String> query = new HashMap<>();

        query.put("date", "1978");

        query.put("subject", "Sea");

        Iterable<String> strings = store.deliverPage(null, null, null, 100, 100);

        Assert.assertNotNull(strings);

        int i = 0;

        for (String s: strings) {

            i++;

            Assert.assertFalse(StringUtils.isEmpty(s));

        }

        Assert.assertEquals(100, i);

        strings = store.deliverPage(null, null, null,100 , (int) (collection.count()-50));

        Assert.assertNotNull(strings);

        i = 0;

        for (String s: strings) {

            i++;

            Assert.assertFalse(StringUtils.isEmpty(s));

        }

        Assert.assertEquals(50, i);

        store.iterate().forEach(it -> Assert.assertFalse(StringUtils.isEmpty(it)));

    }

    @Test

    public void testTruncate() throws Exception {

        final MongoMDStore store = new MongoMDStore("collection", collection, discardedCollection,  MongoTestUtility.generateMDFormat(),metadataRepository);

        feedMDStore(300, store);

        Assert.assertTrue(store.getSize()> 0);

        store.truncate();

        Assert.assertTrue(store.getSize()== 0);

    }

}