D-Net

                        <ARC to="prepareData"/>

                    </ARCS>

                </NODE>

                <NODE name="prepareData" type="SubmitHadoopJob">

                    <DESCRIPTION>index data on ES</DESCRIPTION>

                    <PARAMETERS>

                        <PARAM name="hadoopJob" value="prepareDLIDataJob"/>

                        <PARAM name="cluster" ref="cluster"/>

                        <PARAM name="jobParams">

                            <MAP>

                                <ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>

                                <ENTRY key="mapred.output.dir" ref="workDirObject"/>

                                <ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>

                                <ENTRY key="ontologies" env="ontologies"/>

                            </MAP>

                        </PARAM>

                    </PARAMETERS>

                    <ARCS>

                        <PARAM name="host" value="node0-d-dli.d4science.org"/>

                        <ARC to="indexDataObject"/>

                <NODE name="indexDataObject" type="SubmitHadoopJob">

                    <DESCRIPTION>index data on ES</DESCRIPTION>

                    <PARAMETERS>

                        <PARAM name="hadoopJob" value="elasticsearchFeedJob"/>

                        <PARAM name="cluster" ref="cluster"/>

                        <PARAM name="jobParams">

                            <MAP>

                                <ENTRY key="mapred.input.dir" ref="workDirObject"/>

                            </MAP>

                        </PARAM>

                    </PARAMETERS>

                    <ARCS>

                        <ARC to="indexDataScholix"/>

                    </ARCS>

                </NODE>

                <NODE name="indexDataScholix" type="SubmitHadoopJob">

                        <PARAM name="hadoopJob" value="elasticsearchFeedScholixJob"/>

                                <ENTRY key="mapred.input.dir" ref="workDirObject"/>

package eu.dnetlib.dli.proto;

import java.util.*;

import java.util.function.Function;

import java.util.function.Supplier;

import java.util.stream.Collectors;

import eu.dnetlib.data.proto.*;

import eu.dnetlib.data.proto.KindProtos.Kind;

import static eu.dnetlib.data.proto.DNGFProtos.*;

import static eu.dnetlib.data.proto.dli.DLIObjectProtos.*;

/**

 * Created by sandro on 2/13/17.

 */

//public class DNGFDLIConverter {

//

//    private DNGF mainDNGFEntity;

//

//    private List<DNGF> relations = new ArrayList<>();

//

//    public void setMainEntity(final DNGF mainEntity) {

//        this.mainDNGFEntity = mainEntity;

//    }

//

//    public void addRelation(final DNGF relation) {

//        this.relations.add(relation);

//    }

//

//    public boolean isValid() {

//	    return !relations.isEmpty() && mainDNGFEntity != null && mainDNGFEntity.getKind().equals(Kind.entity);

//    }

//

//    public DLIObject convert() {

//        if (relations.isEmpty() || mainDNGFEntity == null) {

//            throw new IllegalStateException("missing relations or mainEntity");

//        }

//        if (mainDNGFEntity.getKind().equals(KindProtos.Kind.relation)) {

//            throw new IllegalStateException("the main Entity must be an Entity and not a relation");

//        }

//

//        final DNGFEntity entity = mainDNGFEntity.getEntity();

//

//        final DLIObject.Builder builder = makeDLIObject(entity);

//

//

//        this.relations.forEach(r -> manageRelation(r.getRel(), builder));

//

//        //Setting Titles

//        return builder.build();

//    }

//

//    public DLIObjectSummary summaryOf(final DLIObject input) {

//        final DLIObjectSummary.Builder summary = DLIObjectSummary.newBuilder();

//        summary.setId(input.getId());

//        if (input.getMetadata().getTitleList() != null)

//            summary.addAllTitle(input.getMetadata().getTitleList());

//        if (input.getMetadata().getAuthorList() != null)

//            summary.addAllAuthor(input.getMetadata().getAuthorList());

//        if (input.getMetadata().getDateList() != null)

//            summary.addAllDate(input.getMetadata().getDateList());

//

//        summary.setAbstract(input.getMetadata().getAbstract());

//

//        if (input.getMetadata().getSubjectList() != null)

//            summary.addAllSubject(input.getMetadata().getSubjectList());

//

//        if (input.getMetadata().getPublisherList() != null)

//            summary.addAllPublisher(input.getMetadata().getPublisherList());

//

//        if (input.getLocalIdentifierList() != null)

//            summary.addAllLocalIdentifier(input.getLocalIdentifierList());

//

//        final Map<String, CollectedFromType> datasources = new HashMap<>();

//

//        final List<CollectedFromType> collectedFromList = input.getMetadata().getCollectedFromList();

//        if (collectedFromList != null) {

//            collectedFromList.forEach(c -> {

//                if (c.getDatasourceId() != null) {

//                    if (!datasources.containsKey(c.getDatasourceId()))

//                        datasources.put(c.getDatasourceId(), c);

//                    else

//                        datasources.put(c.getDatasourceId(), CollectedFromType.newBuilder(c).setCompletionStatus(CompletionStatus.incomplete).build());

//                }

//            });

//        }

//

//        if (input.getRelationsList() != null) {

//

//            Map<Typology, Integer> countRelations = new HashMap<>();

//            countRelations.put(Typology.publication, 0);

//            countRelations.put(Typology.dataset, 0);

//            countRelations.put(Typology.unknown, 0);

//

//            summary.setTypology(input.getMetadata().getType());

//            input.getRelationsList().forEach(rel -> {

//                Typology t = rel.getTarget().getMetadata().getType();

//                switch (t) {

//                    case publication:

//                        countRelations.put(Typology.publication, countRelations.get(Typology.publication) + 1);

//                        break;

//                    case dataset:

//                        countRelations.put(Typology.dataset, countRelations.get(Typology.dataset) + 1);

//                        break;

//                    case unknown:

//                        countRelations.put(Typology.unknown, countRelations.get(Typology.unknown) + 1);

//                        break;

//                }

//            });

//            summary.setRelatedPublications(countRelations.get(Typology.publication));

//            summary.setRelatedDatasets(countRelations.get(Typology.dataset));

//            summary.setRelatedUnknown(countRelations.get(Typology.unknown));

//

//

//            final List<ResolvedFromType> resolvedFromTypeList = input.getMetadata().getResolvedFromList();

//            resolvedFromTypeList.forEach(r -> {

//                if (r.getDatasourceId() != null) {

//                    datasources.put(r.getDatasourceId(),

//                            CollectedFromType.newBuilder()

//                                    .setDatasourceId(r.getDatasourceId())

//                                    .setDatasourceName(r.getDatasourceName())

//                                    .setCompletionStatus(CompletionStatus.complete)

//                                    .build());

//                }

//            });

//

//

//            summary.addAllDatasources(datasources.values());

//

//

//        }

//

//        return summary.build();

//

//    }

//

//    private DLIObject.Builder makeDLIObject(DNGFEntity entity) {

//        //Set main Identifier

//        final DLIObject.Builder builder = DLIObject.newBuilder();

//        final Set<String> authorList = new HashSet<>();

//        builder.setId(entity.getId());

//

//        //Set all typed Identifier

//        final List<FieldTypeProtos.StructuredProperty> identifiers = entity.getExtension(DliProtos.typedIdentifier);

//        if (identifiers == null || identifiers.isEmpty()) {

//            throw new IllegalStateException(String.format("missing typedIdentifiers on main Entity \n%s", entity));

//

//        }

//

//        identifiers.stream()

//                .collect(

//                        Collectors.groupingBy(s -> String.format("%s::%s", s.getValue().toLowerCase().trim(), s.getQualifier().getClassid().toLowerCase().trim())))

//                .values()

//                .forEach(

//                        v -> builder.addLocalIdentifier(

//                        TypedIdentifier.newBuilder()

//                                .setId(v.get(0).getValue())

//                                .setType(v.get(0).getQualifier().getClassid())

//                ));

//

//        Metadata.Builder dliMetadata = Metadata.newBuilder();

//

//        //ADD Resolved From

//        final List<FieldTypeProtos.KeyValue> resolvedFromList = entity.getExtension(DliProtos.resolvedfrom);

//        if (resolvedFromList != null) {

//            Map<String, FieldTypeProtos.KeyValue> setRFrom = new HashMap<>();

//            resolvedFromList.forEach(rfrom -> setRFrom.put(rfrom.getKey(), rfrom));

//            setRFrom.values().forEach(r -> dliMetadata.addResolvedFrom(

//                    ResolvedFromType.newBuilder()

//                            .setDatasourceId(r.getKey())

//                            .setDatasourceName(r.getValue())

//                            .build()

//            ));

//        }

//

//        final List<FieldTypeProtos.KeyValue> collectedFromList = entity.getCollectedfromList();

//        if (collectedFromList != null) {

//            Map<String, FieldTypeProtos.KeyValue> setCFrom = new HashMap<>();

//            collectedFromList.forEach(cfrom -> setCFrom.put(cfrom.getKey(), cfrom));

//            setCFrom.values().forEach(c -> {

//                final String completionStatus = c.getExtension(DliFieldTypeProtos.completionStatus);

//                final CollectedFromType.Builder cType = CollectedFromType.newBuilder();

//                cType.setDatasourceId(c.getKey());

//                cType.setDatasourceName(c.getValue());

//                cType.setCompletionStatus(CompletionStatus.valueOf(completionStatus));

//                dliMetadata.addCollectedFrom(cType);

//            });

//        }

//

//        final String completionStatus = entity.getExtension(DliProtos.completionStatus);

//        dliMetadata.setCompletionStatus(CompletionStatus.valueOf(completionStatus));

//

//        //populateAuthor(entity, authorList);

//        //Setting Metadata

//        switch (entity.getType()) {

//            case dataset:

//                dliMetadata.setType(Typology.dataset);

//                setMetadataFromDataset(entity.getDataset(), dliMetadata, authorList);

//                break;

//            case publication:

//                dliMetadata.setType(Typology.publication);

//                setMetadataFromPublication(entity.getPublication(), dliMetadata, authorList);

//                break;

//            case unknown:

//                dliMetadata.setType(Typology.unknown);

//                break;

//            default:

//                throw new IllegalStateException(String.format("wrong Type for mainEntity: \n%s", entity));

//        }

//        return builder.setMetadata(dliMetadata);

//    }

//

////    private void populateAuthor(final DNGFEntity entity, final Set<String> authorList) {

////        if (entity.getChildrenList() != null) {

////            entity.getChildrenList().forEach(dedupItem -> {

////                List<PersonProtos.Person> personList = null;

////                List<FieldTypeProtos.StringField> contributorList = null;

////                switch (dedupItem.getType()) {

////                    case publication:

////                        personList = dedupItem.getPublication().getAuthorList();

////                        contributorList = entity.getDataset().getMetadata().getContributorList();

////                        break;

////                    case dataset:

////                        personList = dedupItem.getDataset().getAuthorList();

////                        contributorList = entity.getDataset().getMetadata().getContributorList();

////                        break;

////                }

////                if (contributorList != null) {

////                    contributorList.forEach(c -> authorList.add(c.getValue()));

////                }

////                if (personList != null) {

////                    personList.forEach(person -> {

////                        authorList.add(person.getMetadata().getFullname().getValue());

////                    });

////                }

////            });

////        }

////    }

//

//    private void manageRelation(final DNGFRel relation, final DLIObject.Builder dliObj) {

//        final DLIObject.Builder targetObject = makeDLIObject(relation.getCachedTarget());

//

//        final DLIRelation.Builder relBuilder = DLIRelation.newBuilder()

//		        .setTarget(targetObject)

//		        .setRelationSemantic(SchemeValue.newBuilder().setScheme("datacite").setValue(relation.getRelType().getClassid()));

//

//        final List<FieldTypeProtos.KeyValue> collectedFromList = relation.getCollectedfromList();

//        if (collectedFromList != null) {

//            collectedFromList.forEach(c ->

//                    relBuilder.addCollectedFrom(

//                            CollectedFromType.newBuilder()

//                                    .setDatasourceName(c.getValue())

//                                    .setDatasourceId(c.getKey())

//		                            .setCompletionStatus(CompletionStatus.complete)

//                                    .build()

//                    )

//            );

//        }

//

//        dliObj.addRelations(relBuilder.build());

//    }

//

//

//

//    private void managePublicationDates(final PublicationProtos.Publication publication, final Metadata.Builder metadata) {

//        if (publication.getMetadata().getDateofacceptance() != null) {

//            metadata.addDate(publication.getMetadata().getDateofacceptance().getValue());

//        }

//

//        if (publication.getMetadata().getRelevantdateList() != null && !publication.getMetadata().getRelevantdateList().isEmpty()) {

//            publication.getMetadata().getRelevantdateList().forEach(

//                    date -> metadata.addDate(date.getValue())

//            );

//        }

//    }

//

//    private void manageDatasetDates(final DatasetProtos.Dataset dataset, final Metadata.Builder metadata) {

//        if (dataset.getMetadata().getDateofacceptance() != null) {

//            metadata.addDate(dataset.getMetadata().getDateofacceptance().getValue());

//        }

//

//        if (dataset.getMetadata().getRelevantdateList() != null && !dataset.getMetadata().getRelevantdateList().isEmpty()) {

//            dataset.getMetadata().getRelevantdateList().forEach(

//                    date -> metadata.addDate(date.getValue())

//            );

//        }

//

//    }

//

//    public DNGF getMainDNGFEntity() {

//        return mainDNGFEntity;

//    }

//

//    public void setMainDNGFEntity(DNGF mainDNGFEntity) {

//        this.mainDNGFEntity = mainDNGFEntity;

//    }

//

//    public List<DNGF> getRelations() {

//        return relations;

//    }

//

//    public void setRelations(List<DNGF> relations) {

//        this.relations = relations;

//    }

//}

                    personList.forEach(person -> {

                        authorList.add(person.getMetadata().getFullname().getValue());

                    });