Project

General

Profile

« Previous | Next » 

Revision 48200

[maven-release-plugin] copy for tag dnet-index-solr-service-2.4.3

View differences:

modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/deploy.info
1
{ "type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-service" }
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/solr/feed/SolrDocumentMapperFactory.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import com.google.common.base.Function;
4
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;
5
import eu.dnetlib.functionality.index.model.Any.ValueType;
6
import eu.dnetlib.functionality.index.model.document.IndexDocument;
7
import eu.dnetlib.functionality.index.model.document.Status;
8
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;
9
import eu.dnetlib.functionality.index.utils.MetadataReference;
10
import org.springframework.beans.factory.annotation.Required;
11

  
12
import javax.xml.stream.XMLStreamException;
13
import java.util.Map;
14

  
15
/**
16
 * A factory for creating SolrDocumentMapper objects.
17
 */
18
public class SolrDocumentMapperFactory implements DocumentMapperFactory {
19

  
20
    protected static final String DNETRESULT = "result";
21
    /**
22
     * document factory used for the feed process.
23
     */
24
    private InputDocumentFactory documentFactory;
25

  
26

  
27
    private Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema,
28
                                                            final String dsId,
29
                                                            final String version) {
30

  
31
        return new Function<String, IndexDocument>() {
32

  
33
            @Override
34
            public IndexDocument apply(final String doc) {
35
                SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
36
                try {
37
                    indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT));
38
                } catch (XMLStreamException e) {
39
                    return indexDocument.setMarked();
40
                }
41
                indexDocument.setStatus(Status.OK);
42
                return indexDocument;
43
            }
44
        };
45
    }
46

  
47
    @Override
48
    public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema, final MetadataReference mdRef, final String dsId, final String version, final boolean emptyResult) {
49
        if (emptyResult == false) return getRecordMapper(schema, dsId, version);
50
        else {
51
            return new Function<String, IndexDocument>() {
52

  
53
                @Override
54
                public IndexDocument apply(final String doc) {
55
                    SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
56
                    try {
57

  
58
                        ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {
59
                            @Override
60
                            public String apply(String input) {
61
                                return input;
62
                            }
63
                        };
64
                        indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer));
65
                    } catch (XMLStreamException e) {
66
                        return indexDocument.setMarked();
67
                    }
68
                    indexDocument.setStatus(Status.OK);
69
                    return indexDocument;
70
                }
71
            };
72
        }
73
    }
74

  
75
    /**
76
     * Gets the document factory.
77
     *
78
     * @return the documentFactory
79
     */
80
    public InputDocumentFactory getDocumentFactory() {
81
        return documentFactory;
82
    }
83

  
84
    /**
85
     * Sets the document factory.
86
     *
87
     * @param documentFactory the documentFactory to set
88
     */
89
    @Required
90
    public void setDocumentFactory(final InputDocumentFactory documentFactory) {
91
        this.documentFactory = documentFactory;
92
    }
93

  
94
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/solr/browsing/BrowsingRow.java
1
package eu.dnetlib.functionality.index.solr.browsing;
2

  
3
import java.util.List;
4

  
5
import javax.xml.bind.annotation.XmlAccessType;
6
import javax.xml.bind.annotation.XmlAccessorType;
7
import javax.xml.bind.annotation.XmlElement;
8
import javax.xml.bind.annotation.XmlRootElement;
9

  
10
import org.springframework.beans.factory.annotation.Required;
11

  
12
import eu.dnetlib.data.provision.index.rmi.GroupResult;
13

  
14
/**
15
 *
16
 * serialization of the browsing result.
17
 *
18
 * <row> <groupresult field="facetFieldName1"> <value>facetFieldValue</value> <count>1</count> </groupresult>
19
 * 
20
 * <groupresult field="facetFieldName2"> <value>facetFieldValue</value> <count>1</count> </groupresult>
21
 * 
22
 * </row>
23
 *
24
 * @author claudio
25
 *
26
 */
27
@XmlRootElement(namespace = "", name = "row")
28
@XmlAccessorType(XmlAccessType.FIELD)
29
public class BrowsingRow {
30

  
31
	@XmlElement(name = "groupresult", required = true)
32
	private List<GroupResult> groupresult;
33

  
34
	public BrowsingRow() {}
35

  
36
	public BrowsingRow(final List<GroupResult> groupresult) {
37
		this.groupresult = groupresult;
38
	}
39

  
40
	/**
41
	 * adds a GroupResult.
42
	 *
43
	 * @param fieldName
44
	 * @param fieldValue
45
	 * @param count
46
	 */
47
	public void addBrowsingRow(final String fieldName, final String fieldValue, final int count) {
48
		groupresult.add(new GroupResult(fieldName, fieldValue, count));
49
	}
50

  
51
	@Override
52
	public boolean equals(final Object obj) {
53

  
54
		if (!(obj instanceof BrowsingRow)) return false;
55

  
56
		final BrowsingRow brws = (BrowsingRow) obj;
57

  
58
		return groupresult.equals(brws.getGroupResult());
59
	}
60

  
61
	public List<GroupResult> getGroupResult() {
62
		return groupresult;
63
	}
64

  
65
	@Required
66
	public void setGroupResult(final List<GroupResult> groupresult) {
67
		this.groupresult = groupresult;
68
	}
69

  
70
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/solr/browsing/GroupResult.java
1
package eu.dnetlib.functionality.index.solr.browsing;
2

  
3
import javax.xml.bind.annotation.XmlAttribute;
4
import javax.xml.bind.annotation.XmlRootElement;
5

  
6
import org.springframework.beans.factory.annotation.Required;
7

  
8
/**
9
 * <pre>
10
 * {@code
11
 * <groupresult field="facetFieldName">
12
 * 	<value>facetFieldValue</value>
13
 * 	<count>1</count>
14
 * </groupresult>
15
 * }
16
 * </pre>
17
 *
18
 * @author claudio
19
 *
20
 */
21
@XmlRootElement(namespace = "", name = "groupresult")
22
public class GroupResult {
23

  
24
	private String name;
25

  
26
	private String value;
27

  
28
	private int count;
29

  
30
	public GroupResult() {}
31

  
32
	/**
33
	 * Builds a groupResult.
34
	 *
35
	 * @param fieldName
36
	 * @param fieldValue
37
	 * @param count
38
	 */
39
	public GroupResult(final String name, final String value, final int count) {
40
		this.name = name;
41
		this.value = value;
42
		this.count = count;
43
	}
44

  
45
	@Override
46
	public boolean equals(final Object obj) {
47
		if (!(obj instanceof GroupResult)) return false;
48
		final GroupResult g = (GroupResult) obj;
49
		if ((this.getCount() == g.getCount()) && this.getName().equals(g.getName()) && this.getValue().equals(g.getValue())) return true;
50
		return false;
51
	}
52

  
53
	@XmlAttribute(name = "field", required = true)
54
	public String getName() {
55
		return name;
56
	}
57

  
58
	public String getValue() {
59
		return value;
60
	}
61

  
62
	public int getCount() {
63
		return count;
64
	}
65

  
66
	@Required
67
	public void setName(final String name) {
68
		this.name = name;
69
	}
70

  
71
	@Required
72
	public void setValue(final String value) {
73
		this.value = value;
74
	}
75

  
76
	@Required
77
	public void setCount(final int count) {
78
		this.count = count;
79
	}
80

  
81
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/utils/IndexSchemaFactory.java
1
package eu.dnetlib.functionality.index.utils;
2

  
3
import java.io.IOException;
4

  
5
import javax.xml.transform.Transformer;
6
import javax.xml.transform.TransformerConfigurationException;
7
import javax.xml.transform.TransformerException;
8
import javax.xml.transform.TransformerFactory;
9
import javax.xml.transform.TransformerFactoryConfigurationError;
10

  
11
import org.apache.commons.logging.Log;
12
import org.apache.commons.logging.LogFactory;
13
import org.dom4j.Document;
14
import org.dom4j.DocumentException;
15
import org.dom4j.io.DocumentResult;
16
import org.dom4j.io.DocumentSource;
17
import org.dom4j.io.SAXReader;
18
import org.springframework.beans.factory.annotation.Required;
19
import org.springframework.core.io.Resource;
20

  
21
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
22

  
23
public class IndexSchemaFactory {
24

  
25
	private static final Log log = LogFactory.getLog(IndexSchemaFactory.class); // NOPMD by marko on 11/24/08 5:02 PM
26

  
27
	private final static String TEXT_FIELD_TYPE = "textFieldType";
28

  
29
	private Resource schemaTemplate;
30

  
31
	private String textFieldType;
32

  
33
	/**
34
	 * Transformer used to get the index schema.
35
	 */
36
	private Transformer transformer;
37

  
38
	public void init() throws TransformerConfigurationException, TransformerFactoryConfigurationError, DocumentException, IOException {
39

  
40
		transformer = TransformerFactory.newInstance().newTransformer(new DocumentSource(new SAXReader().read(getSchemaTemplate().getInputStream())));
41
		transformer.setParameter(TEXT_FIELD_TYPE, getTextFieldType());
42
	}
43

  
44
	/**
45
	 * @param docIn
46
	 *            input document
47
	 * @return the application of the schemaXslt transformation to the docIn document
48
	 * @throws IndexServiceException
49
	 */
50
	public String getSchema(final Document fields) throws IndexServiceException {
51

  
52
		log.debug("default field type: " + getTextFieldType());
53
		log.debug("building schema from fields: \n" + fields.asXML() + "\n");
54

  
55
		final DocumentResult result = new DocumentResult();
56
		try {
57
			transformer.transform(new DocumentSource(fields), result);
58
			String xml = result.getDocument().asXML();
59

  
60
			log.debug("new index schema:\n" + xml);
61

  
62
			return xml;
63
		} catch (TransformerException e) {
64
			throw new IndexServiceException(e);
65
		}
66
	}
67

  
68
	@Required
69
	public void setSchemaTemplate(final Resource schemaTemplate) {
70
		this.schemaTemplate = schemaTemplate;
71
	}
72

  
73
	public Resource getSchemaTemplate() {
74
		return schemaTemplate;
75
	}
76

  
77
	@Required
78
	public void setTextFieldType(final String textFieldType) {
79
		this.textFieldType = textFieldType;
80
	}
81

  
82
	public String getTextFieldType() {
83
		return textFieldType;
84
	}
85

  
86
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/utils/IndexConfigFactory.java
1
package eu.dnetlib.functionality.index.utils;
2

  
3
import java.util.Map;
4
import java.util.Map.Entry;
5

  
6
import org.antlr.stringtemplate.StringTemplate;
7
import org.apache.commons.logging.Log;
8
import org.apache.commons.logging.LogFactory;
9
import org.springframework.beans.factory.annotation.Required;
10

  
11
import com.google.common.base.Predicate;
12
import com.google.common.collect.Maps;
13

  
14
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
15

  
16
public class IndexConfigFactory {
17

  
18
	private static final Log log = LogFactory.getLog(IndexConfigFactory.class); // NOPMD
19
																				// by
20
																				// marko
21
																				// on
22
																				// 11/24/08
23
																				// 5:02
24
																				// PM
25

  
26
	public enum CONFIG_PARAMS {
27
		// TODO replace with actual values
28
		luceneMatchVersion,
29
		serverLibPath,
30
		filterCacheSize,
31
		filterCacheInitialSize,
32
		queryCacheSize,
33
		queryCacheInitialSize,
34
		documentCacheSize,
35
		documentCacheInitialSize,
36
		queryResultWindowSize,
37
		queryResultMaxDocCached,
38
		ramBufferSizeMB,
39
		mergeFactor,
40
		termIndexInterval,
41
		autosoftcommit,
42
		autocommit,
43
		maxIndexingThreads;
44

  
45
		static boolean isValid(final String v) {
46
			try {
47
				CONFIG_PARAMS.valueOf(v);
48
				return true;
49
			} catch (Throwable e) {
50
				return false;
51
			}
52
		}
53
	};
54

  
55
	/**
56
	 * Index configuration template.
57
	 */
58
	private StringTemplate solrConfig;
59

  
60
	/**
61
	 * @param docIn
62
	 *            input document
63
	 * @return the application of the schemaXslt transformation to the docIn
64
	 *         document
65
	 * @throws IndexServiceException
66
	 */
67
	public String getConfig(final Map<String, String> params)
68
			throws IndexServiceException {
69
		final StringTemplate conf = new StringTemplate(getSolrConfig()
70
				.getTemplate());
71
		for (Entry<String, String> e : filter(params).entrySet()) {
72
			log.debug("setting conf property [" + e.getKey() + ":"
73
					+ e.getValue() + "]");
74
			conf.setAttribute(e.getKey(), e.getValue());
75
		}
76
		return conf.toString();
77
	}
78

  
79
	private Map<String, String> filter(final Map<String, String> params) {
80
		return Maps.filterKeys(params, new Predicate<String>() {
81

  
82
			@Override
83
			public boolean apply(final String input) {
84
				return CONFIG_PARAMS.isValid(input);
85
			}
86
		});
87
	}
88

  
89
	public StringTemplate getSolrConfig() {
90
		return solrConfig;
91
	}
92

  
93
	@Required
94
	public void setSolrConfig(final StringTemplate solrConfig) {
95
		this.solrConfig = solrConfig;
96
	}
97

  
98
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/utils/ZkUtils.java
1
package eu.dnetlib.functionality.index.utils;
2

  
3
import java.io.IOException;
4
import java.io.InputStream;
5
import java.util.Map;
6
import java.util.Map.Entry;
7

  
8
import org.apache.commons.io.IOUtils;
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11
import org.apache.solr.common.cloud.SolrZkClient;
12
import org.apache.zookeeper.CreateMode;
13
import org.apache.zookeeper.KeeperException;
14
import org.dom4j.Document;
15
import org.springframework.beans.factory.annotation.Autowired;
16
import org.springframework.beans.factory.annotation.Required;
17
import org.springframework.core.io.Resource;
18
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
19

  
20
import com.google.common.collect.Maps;
21

  
22
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
23

  
24
/**
25
 * The Class ZkUtils.
26
 */
27
public class ZkUtils {
28

  
29
	/**
30
	 * The log.
31
	 */
32
	private static final Log log = LogFactory.getLog(ZkUtils.class); // NOPMD by marko on 11/24/08 5:02 PM
33

  
34
	/** The Constant CONFIGS_PATH. */
35
	private static final String CONFIGS_PATH = "/configs";
36

  
37
	/** The schema factory. */
38
	@Autowired
39
	private IndexSchemaFactory schemaFactory;
40

  
41
	/** The config factory. */
42
	private IndexConfigFactory configFactory;
43

  
44
	/** The static configuration classpath. */
45
	private String staticConfigurationClasspath;
46

  
47
	/**
48
	 * Upload zookeper config.
49
	 * 
50
	 * @param zkClient
51
	 *            the zk client
52
	 * @param coreName
53
	 *            the core name
54
	 * @param fields
55
	 *            the fields
56
	 * @param params
57
	 *            the params
58
	 * @param overwrite
59
	 *            the overwrite
60
	 */
61
	public void uploadZookeperConfig(final SolrZkClient zkClient,
62
			final String coreName,
63
			final Document fields,
64
			final Map<String, String> params,
65
			final boolean overwrite) {
66

  
67
		final String basepath = CONFIGS_PATH + "/" + coreName;
68

  
69
		log.info("uploading solr configuration to ZK for index collection: " + coreName);
70

  
71
		try {
72
			if (overwrite) {
73
				log.info("cleanup ZK configuration: " + coreName);
74
				for (String child : zkClient.getSolrZooKeeper().getChildren(basepath, false)) {
75
					final String path = basepath + "/" + child;
76
					log.debug("cleanup ZK file: " + path);
77
					zkClient.delete(path, -1, true);
78
				}
79
				zkClient.delete(basepath, -1, true);
80
			}
81
			if (!zkClient.exists(basepath, true)) {
82
				log.info("upload ZK configuration: " + coreName);
83
				zkClient.makePath(basepath, true);
84
				uploadConfiguration(zkClient, basepath, buildConfiguration(coreName, fields, params));
85
			}
86
			log.info("upload ZK configuration complete");
87
		} catch (Exception e) {
88
			log.error("unable to upload solr configuration", e);
89
		}
90
	}
91

  
92
	/**
93
	 * Builds the configuration.
94
	 * 
95
	 * @param indexName
96
	 *            the index name
97
	 * @param fields
98
	 *            the fields
99
	 * @param params
100
	 *            the params
101
	 * @return the map
102
	 * @throws IndexServiceException
103
	 *             the index service exception
104
	 */
105
	private Map<String, byte[]> buildConfiguration(final String indexName, final Document fields, final Map<String, String> params)
106
			throws IndexServiceException {
107

  
108
		Map<String, byte[]> res = Maps.newHashMap();
109

  
110
		try {
111
			log.debug("adding schema.xml to the resource map");
112
			res.put("schema.xml", schemaFactory.getSchema(fields).getBytes());
113
			
114
			res.put("solrconfig.xml", configFactory.getConfig(params).getBytes());
115
			log.debug("adding solrconfig.xml to the resource map");
116
			PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
117
			Resource[] resources = resolver.getResources(getStaticConfigurationClasspath());
118

  
119
			for (Resource r : resources) {
120
				InputStream is = r.getInputStream();
121
				if ((r.getFilename() != null) && !r.getFilename().isEmpty()) {
122
				    res.put(r.getFilename(), IOUtils.toByteArray(is));
123
				    log.debug("adding " + r.getFilename() +" to the resource map");
124
				    is.close();
125
				}
126
			}
127
			return res;
128
		} catch (Throwable e) {
129
			throw new IndexServiceException("failed to build configuration", e);
130
		}
131
	}
132

  
133
	/**
134
	 * Upload configuration.
135
	 * 
136
	 * @param zkClient
137
	 *            the zk client
138
	 * @param basePath
139
	 *            the base path
140
	 * @param resources
141
	 *            the resources
142
	 * @throws KeeperException
143
	 *             the keeper exception
144
	 * @throws InterruptedException
145
	 *             the interrupted exception
146
	 * @throws IOException
147
	 *             Signals that an I/O exception has occurred.
148
	 */
149
	private void uploadConfiguration(final SolrZkClient zkClient, final String basePath, final Map<String, byte[]> resources) throws KeeperException,
150
			InterruptedException, IOException {
151

  
152
		if (!zkClient.exists(basePath, true)) {
153
			zkClient.makePath(basePath, true);
154
		}
155

  
156
		for (final Entry<String, byte[]> e : resources.entrySet()) {
157
			String path = basePath + "/" + e.getKey();
158
			log.debug("upload ZK configuration: " + path);
159
			zkClient.create(path, e.getValue(), CreateMode.PERSISTENT, true);
160
		}
161
	}
162

  
163
	/**
164
	 * Gets the config factory.
165
	 * 
166
	 * @return the config factory
167
	 */
168
	public IndexConfigFactory getConfigFactory() {
169
		return configFactory;
170
	}
171

  
172
	/**
173
	 * Sets the config factory.
174
	 * 
175
	 * @param configFactory
176
	 *            the new config factory
177
	 */
178
	@Required
179
	public void setConfigFactory(final IndexConfigFactory configFactory) {
180
		this.configFactory = configFactory;
181
	}
182

  
183
	/**
184
	 * Gets the static configuration classpath.
185
	 * 
186
	 * @return the static configuration classpath
187
	 */
188
	public String getStaticConfigurationClasspath() {
189
		return staticConfigurationClasspath;
190
	}
191

  
192
	/**
193
	 * Sets the static configuration classpath.
194
	 * 
195
	 * @param staticConfigurationClasspath
196
	 *            the new static configuration classpath
197
	 */
198
	@Required
199
	public void setStaticConfigurationClasspath(final String staticConfigurationClasspath) {
200
		this.staticConfigurationClasspath = staticConfigurationClasspath;
201
	}
202

  
203
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/utils/RemoteSolrAdministrator.java
1
package eu.dnetlib.functionality.index.utils;
2

  
3
import java.io.IOException;
4
import java.util.Map;
5

  
6
import com.google.common.collect.Maps;
7
import com.google.gson.JsonElement;
8
import com.google.gson.JsonObject;
9
import com.google.gson.JsonParser;
10
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
11
import eu.dnetlib.functionality.index.model.Any.ValueType;
12
import org.apache.commons.logging.Log;
13
import org.apache.commons.logging.LogFactory;
14
import org.apache.http.HttpResponse;
15
import org.apache.http.client.HttpClient;
16
import org.apache.http.client.methods.HttpGet;
17
import org.apache.solr.client.solrj.SolrServer;
18
import org.apache.solr.client.solrj.SolrServerException;
19
import org.apache.solr.client.solrj.impl.CloudSolrServer;
20
import org.apache.solr.client.solrj.request.LukeRequest;
21
import org.apache.solr.client.solrj.response.LukeResponse;
22
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
23
import org.apache.solr.client.solrj.response.SolrPingResponse;
24
import org.apache.solr.common.cloud.SolrZkClient;
25
import org.apache.zookeeper.WatchedEvent;
26
import org.apache.zookeeper.Watcher;
27
import org.apache.zookeeper.data.Stat;
28
import org.springframework.beans.factory.annotation.Required;
29

  
30
public class RemoteSolrAdministrator {
31

  
32
	/**
33
	 * The log.
34
	 */
35
	private static final Log log = LogFactory.getLog(RemoteSolrAdministrator.class);
36

  
37
	/** The create url request. */
38
	private static String createURLRequest = "http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&maxShardsPerNode=%s&collection.configName=%s";
39

  
40
	/** The create url request. */
41
	private static String reloadURLRequest = "http://%s:%s/solr/admin/collections?action=RELOAD&name=%s";
42

  
43
	/** The http client. */
44
	private HttpClient httpClient;
45

  
46
	protected Map<String, Map<String, ValueType>> cachedSchema;
47

  
48
	public RemoteSolrAdministrator() {
49
		this.cachedSchema = Maps.newHashMap();
50
	}
51

  
52
	public boolean createSolrIndex(final String host,
53
			final String port,
54
			final String collectionName,
55
			final String numShard,
56
			final String replicationFactor,
57
			final String maxShardsPerNode,
58
		final String collectionConfigName) throws IndexServiceException {
59

  
60
		final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);
61
		log.info(uri);
62
		HttpGet request = new HttpGet(uri);
63
		HttpResponse response;
64
		try {
65
			response = getHttpClient().execute(request);
66

  
67
		} catch (Exception e) {
68
			throw new IndexServiceException("Unable to send request to solr server", e);
69
		} finally {
70
			request.releaseConnection();
71
		}
72
		if (response.getStatusLine().getStatusCode() != 200)
73
			throw new IndexServiceException("Error on creating index the error code from solr is" + response.toString());
74
		return false;
75
	}
76

  
77
	public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrServer server) {
78

  
79
		Watcher watcher = new Watcher() {
80

  
81
			@Override
82
			public void process(final WatchedEvent arg0) {}
83
		};
84
		try {
85
			// server.connect();
86
			SolrZkClient client = server.getZkStateReader().getZkClient();
87
			if (!client.isConnected()) {
88
				server.shutdown();
89
				server.connect();
90
			}
91

  
92
			byte[] data = client.getData("/clusterstate.json", watcher, new Stat(), true);
93
			if (data == null) return false;
94
			String jsonLine = new String(data);
95
			JsonElement jelement = new JsonParser().parse(jsonLine);
96
			JsonObject jobject = jelement.getAsJsonObject();
97
			if (jobject.has(indexCollectionId)) {
98
				server.setDefaultCollection(indexCollectionId);
99
				server.connect();
100
				SolrPingResponse status = server.ping();
101
				return (Integer) status.getResponseHeader().get("status") == 0;
102
			} else return false;
103

  
104
		} catch (Exception e) {
105
			log.error(e);
106
			return false;
107
		}
108
	}
109

  
110
	private ValueType resolveSolrTypeClassName(final String solrTypeName) {
111
		if (solrTypeName.contains("LongField")) return ValueType.LONG;
112
		else if (solrTypeName.contains("IntField")) return ValueType.LONG;
113
		else if (solrTypeName.contains("short")) return ValueType.LONG;
114
		else if (solrTypeName.contains("float")) return ValueType.DOUBLE;
115
		else if (solrTypeName.contains("double")) return ValueType.DOUBLE;
116
		else if (solrTypeName.contains("date")) return ValueType.DATETIME;
117
		else return ValueType.STRING;
118
	}
119

  
120
	public Map<String, ValueType> getFieldNamesAndTypes(final String coreName, final SolrServer server) throws IndexServiceException {
121
		try {
122
			if (cachedSchema.get(coreName) == null) {
123
				synchronized (cachedSchema) {
124
					Map<String, ValueType> schema = readFieldNamesAndTypes(coreName, server);
125
					log.info("setting cache for schema of collection: " + coreName);
126
					cachedSchema.put(coreName, schema);
127
				}
128
			}
129
			return cachedSchema.get(coreName);
130
		} catch (Exception e) {
131
			throw new IndexServiceException("Unable to get Schema for " + coreName + " exception", e);
132
		}
133
	}
134

  
135
	private Map<String, ValueType> readFieldNamesAndTypes(final String coreName, final SolrServer server) throws SolrServerException, IOException {
136
		final LukeRequest request = new LukeRequest();
137
		request.setShowSchema(true);
138
		request.setNumTerms(0);
139
		final LukeResponse response = request.process(server);
140
		final Map<String, FieldInfo> fieldInfos = response.getFieldInfo();
141
		final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo();
142
		final Map<String, ValueType> result = Maps.newHashMap();
143
		for (FieldInfo fieldInfo : fieldInfos.values()) {
144
			LukeResponse.FieldTypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldInfo.getType());
145
			final String fieldName = fieldTypeInfo.getName().toLowerCase();
146
			final ValueType fieldType = resolveSolrTypeClassName(fieldName);
147
			result.put(fieldInfo.getName(), fieldType);
148
		}
149
		return result;
150
	}
151

  
152
	private String generateCreateIndexRequest(final String host,
153
			final String port,
154
			final String collectionName,
155
			final String numShard,
156
			final String replicationFactor,
157
			final String collectionConfigName,
158
			final String maxShardsPerNode) {
159
		return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);
160
	}
161

  
162
	private String generateUpdateIndexRequest(final String host, final String port, final String collectionName) {
163
		return String.format(reloadURLRequest, host, port, collectionName);
164
	}
165

  
166
	/**
167
	 * @return the httpClient
168
	 */
169
	public HttpClient getHttpClient() {
170
		return httpClient;
171
	}
172

  
173
	/**
174
	 * @param httpClient
175
	 *            the httpClient to set
176
	 */
177
	@Required
178
	public void setHttpClient(final HttpClient httpClient) {
179
		log.info("setting http client " + httpClient.getClass());
180
		this.httpClient = httpClient;
181
	}
182

  
183
	public void reloadCollection(final String host, final String port, final String collectionName) throws IndexServiceException {
184
		log.info("creating the request of reload index " + generateUpdateIndexRequest(host, port, collectionName));
185
		HttpGet request = new HttpGet(generateUpdateIndexRequest(host, port, collectionName));
186
		HttpResponse response;
187
		try {
188
			response = httpClient.execute(request);
189
		} catch (Exception e) {
190
			throw new IndexServiceException("Unable to send request to solr server", e);
191
		} finally {
192
			request.releaseConnection();
193
		}
194
		if (response.getStatusLine().getStatusCode() != 200)
195
			throw new IndexServiceException("Error on reloading index the error code from solr is" + response.toString());
196

  
197
	}
198

  
199
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/SolrIndexServerDAO.java
1
package eu.dnetlib.functionality.index;
2

  
3
import java.io.StringReader;
4
import java.util.Map;
5

  
6
import eu.dnetlib.functionality.cql.CqlValueTransformerMap;
7
import org.apache.commons.logging.Log;
8
import org.apache.commons.logging.LogFactory;
9
import org.apache.solr.client.solrj.impl.CloudSolrServer;
10
import org.dom4j.Document;
11
import org.dom4j.DocumentException;
12
import org.dom4j.io.SAXReader;
13
import org.springframework.beans.factory.annotation.Autowired;
14
import org.springframework.beans.factory.annotation.Required;
15

  
16
import com.google.common.collect.Maps;
17

  
18
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
19

  
20
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;
21
import eu.dnetlib.functionality.index.model.Any.ValueType;
22
import eu.dnetlib.functionality.index.query.IndexQueryFactory;
23
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;
24
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory;
25
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;
26
import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory;
27
import eu.dnetlib.functionality.index.utils.IndexConfigFactory;
28
import eu.dnetlib.functionality.index.utils.MetadataReference;
29
import eu.dnetlib.functionality.index.utils.RemoteSolrAdministrator;
30
import eu.dnetlib.functionality.index.utils.ZkUtils;
31

  
32
/**
33
 * The Class SolrIndexServerDAO.
34
 */
35
public class SolrIndexServerDAO extends AbstractBackendDescriptor implements IndexServerDAO {
36

  
37
	/**
38
	 * The log.
39
	 */
40
	private static final Log log = LogFactory.getLog(SolrIndexServerDAO.class); // NOPMD by marko on 11/24/08 5:02 PM
41

  
42
	/** The zk utils. */
43
	@Autowired
44
	private ZkUtils zkUtils;
45

  
46
	/** The query response factory. */
47
	@Autowired
48
	private SolrIndexQueryResponseFactory queryResponseFactory;
49

  
50
	@Autowired
51
	private SolrIndexQueryFactory solrIndexQueryFactory;
52

  
53
	/** The solr document mapper factory. */
54
	@Autowired
55
	private SolrDocumentMapperFactory solrDocumentMapperFactory;
56

  
57
	/** The solr administrator. */
58
	private RemoteSolrAdministrator solrAdministrator;
59

  
60
	/** The solr type based cql value transformer map. */
61
	@Autowired
62
	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;
63

  
64
	/**
65
	 * {@inheritDoc}
66
	 * 
67
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#createIndexCollection(eu.dnetlib.functionality.index.utils.MetadataReference,
68
	 *      java.lang.String)
69
	 */
70
	@Override
71
	public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {
72
		CloudSolrServer server = null;
73
		try {
74

  
75
			server = getServer();
76
			server.connect();
77

  
78
			if (!solrAdministrator.indexCollectionExists(mdref.toString(), server)) {
79
				Map<String, String> params = Maps.newHashMap();
80

  
81
				final Map<String, String> p = getServiceProperties();
82
				params.put("numShards", p.get("numShards"));
83
				params.put("replicationFactor", p.get("replicationFactor"));
84

  
85
				for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {
86
					params.put(param_Name.toString(), p.get(param_Name.toString()));
87
				}
88

  
89
				zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, false);
90
				solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),
91
						mdref.toString(), p.get("maxShardsPerNode"));
92
			}
93
			server.getZkStateReader().close();
94

  
95
		} catch (Exception e) {
96
			log.error("Error on creating IndexCollection", e);
97
			throw new IndexServiceException("Error on creating IndexCollection", e);
98
		} finally {
99
			if (server != null) {
100
				server.shutdown();
101
			}
102
		}
103
	}
104

  
105
	@Override
106
	public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {
107
		CloudSolrServer server = null;
108
		try {
109
			server = getServer();
110
			server.connect();
111
			Map<String, String> params = Maps.newHashMap();
112

  
113
			params.put("numShards", getServiceProperties().get("numShards"));
114
			params.put("replicationFactor", getServiceProperties().get("replicationFactor"));
115

  
116
			for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {
117
				params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString()));
118
			}
119

  
120
			zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true);
121
			server.getZkStateReader().close();
122
			server.shutdown();
123
			solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());
124

  
125
		} catch (Exception e) {
126
			log.error("Error on updating IndexCollection", e);
127
			throw new IndexServiceException("Error on updating IndexCollection", e);
128
		} finally {
129
			if (server != null) {
130
				server.shutdown();
131
			}
132
		}
133
	}
134

  
135
	/**
136
	 * Parses the fields parameter.
137
	 * 
138
	 * @param fields
139
	 *            the fields
140
	 * @return the document
141
	 * @throws IndexServiceException
142
	 *             the index service exception
143
	 */
144
	private Document parse(final String fields) throws IndexServiceException {
145
		try {
146
			return new SAXReader().read(new StringReader(fields));
147
		} catch (DocumentException e) {
148
			throw new IndexServiceException("unable to parse fields: " + fields, e);
149
		}
150
	}
151

  
152
	/**
153
	 * {@inheritDoc}
154
	 * 
155
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getIndexCollection(eu.dnetlib.functionality.index.utils.MetadataReference)
156
	 */
157
	@Override
158
	public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {
159
		CloudSolrServer newServer = getServer(mdref);
160
		return new SolrIndexCollection(newServer);
161
	}
162

  
163
	/**
164
	 * {@inheritDoc}
165
	 * 
166
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getSchema(MetadataReference)
167
	 */
168
	@Override
169
	public Map<String, ValueType> getSchema(final MetadataReference mdRef) throws IndexServiceException {
170
		CloudSolrServer server = getServer(mdRef);
171
		Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), server);
172
		server.shutdown();
173
		return fields;
174
	}
175

  
176
	/**
177
	 * {@inheritDoc}
178
	 * 
179
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getCqlValueTransformerMap(MetadataReference)
180
	 */
181
	@Override
182
	public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexServiceException {
183
		return tMapFactory.getIt(getSchema(mdRef));
184
	}
185

  
186
	/**
187
	 * {@inheritDoc}
188
	 * 
189
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getDocumentMapperFactory()
190
	 */
191
	@Override
192
	public DocumentMapperFactory getDocumentMapperFactory() throws IndexServiceException {
193
		return solrDocumentMapperFactory;
194
	}
195

  
196
	/**
197
	 * {@inheritDoc}
198
	 * 
199
	 * @see eu.dnetlib.functionality.index.IndexServerDAO#shutdown(MetadataReference)
200
	 */
201
	@Override
202
	public void shutdown(final MetadataReference mdRef) throws IndexServiceException {
203
		getServer(mdRef).shutdown();
204

  
205
	}
206

  
207
	/**
208
	 * Gets a server with the default collection set according to the given mdRef.
209
	 * 
210
	 * @param mdRef
211
	 *            the md ref
212
	 * @return a server instance
213
	 * @throws IndexServiceException
214
	 *             the index service exception
215
	 */
216
	private CloudSolrServer getServer(final MetadataReference mdRef) throws IndexServiceException {
217
		CloudSolrServer server = getServer();
218
		server.setDefaultCollection(mdRef.toString());
219
		return server;
220
	}
221

  
222
	/**
223
	 * Gets the server.
224
	 * 
225
	 * @return a server instance
226
	 */
227
	private CloudSolrServer getServer() {
228
		String address = getEndpoint().get(ADDRESS);
229
		log.info("connecting to address: " + address);
230
		return new CloudSolrServer(address);
231

  
232
	}
233

  
234
	/**
235
	 * Gets the solr administrator.
236
	 * 
237
	 * @return the solrAdministrator
238
	 */
239
	public RemoteSolrAdministrator getSolrAdministrator() {
240
		return solrAdministrator;
241
	}
242

  
243
	/**
244
	 * Sets the solr administrator.
245
	 * 
246
	 * @param solrAdministrator
247
	 *            the solrAdministrator to set
248
	 */
249
	@Required
250
	public void setSolrAdministrator(final RemoteSolrAdministrator solrAdministrator) {
251
		this.solrAdministrator = solrAdministrator;
252
	}
253

  
254
	@Override
255
	public IndexQueryFactory getIndexQueryFactory() {
256
		return solrIndexQueryFactory;
257
	}
258

  
259
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/java/eu/dnetlib/functionality/index/SolrIndexCollection.java
1
package eu.dnetlib.functionality.index;
2

  
3
import java.util.Collection;
4
import java.util.Iterator;
5

  
6
import org.apache.commons.lang.StringUtils;
7
import org.apache.commons.logging.Log;
8
import org.apache.commons.logging.LogFactory;
9
import org.apache.solr.client.solrj.impl.CloudSolrServer;
10
import org.apache.solr.client.solrj.response.UpdateResponse;
11
import org.apache.solr.common.SolrInputDocument;
12

  
13
import com.google.common.base.Function;
14
import com.google.common.collect.Iterators;
15
import com.google.common.collect.Lists;
16

  
17
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
18
import eu.dnetlib.functionality.index.model.document.IndexDocument;
19
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;
20
import eu.dnetlib.functionality.index.utils.IndexFieldUtility;
21

  
22
/**
23
 * The Class SolrIndexCollection.
24
 */
25
public class SolrIndexCollection implements IndexCollection {
26

  
27
	/**
28
	 * The log.
29
	 */
30
	private static final Log log = LogFactory.getLog(SolrIndexCollection.class); // NOPMD by marko on 11/24/08 5:02 PM
31

  
32
	/** The Constant STATUS_INDEX_OK. */
33
	public static final int STATUS_INDEX_OK = 0;
34

  
35
	/** The server. */
36
	private CloudSolrServer server;
37

  
38
	private boolean shutdown = false;
39

  
40
	/**
41
	 * The Constructor.
42
	 *
43
	 * @param newServer
44
	 *            the server
45
	 * @param queryResponseFactory
46
	 *            the query response factory
47
	 */
48
	public SolrIndexCollection(final CloudSolrServer newServer) {
49
		this.server = newServer;
50
		server.connect();
51
	}
52

  
53
	/**
54
	 * {@inheritDoc}
55
	 *
56
	 * @see eu.dnetlib.functionality.index.IndexCollection#add(eu.dnetlib.functionality.index.model.document.IndexDocument)
57
	 */
58
	@Override
59
	public boolean add(final IndexDocument doc) throws IndexServiceException {
60
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
61
		final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;
62
		try {
63
			final UpdateResponse response = server.add(solrDocument.getSolrDocument());
64
			return response.getStatus() == 0;
65
		} catch (final Exception e) {
66
			throw new IndexServiceException("Unable to add document", e);
67
		}
68
	}
69

  
70
	/**
71
	 * {@inheritDoc}
72
	 *
73
	 * @see eu.dnetlib.functionality.index.IndexCollection#addAll(java.util.Iterator)
74
	 */
75
	@Override
76
	public boolean addAll(final Iterator<IndexDocument> docs) throws IndexServiceException {
77
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
78
		final Iterator<SolrInputDocument> solrDocs = Iterators.transform(docs, new Function<IndexDocument, SolrInputDocument>() {
79

  
80
			@Override
81
			public SolrInputDocument apply(final IndexDocument doc) {
82
				final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;
83
				return solrDocument.getSolrDocument();
84
			}
85
		});
86

  
87
		try {
88
			final UpdateResponse response = server.add(Lists.newArrayList(solrDocs));
89
			return response.getStatus() == 0;
90
		} catch (final Exception e) {
91
			throw new IndexServiceException("Unable to add document", e);
92
		}
93
	}
94

  
95
	/**
96
	 * {@inheritDoc}
97
	 *
98
	 * @see eu.dnetlib.functionality.index.IndexCollection#addAll(java.util.Collection)
99
	 */
100
	@Override
101
	public boolean addAll(final Collection<IndexDocument> docs) throws IndexServiceException {
102
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
103
		return addAll(docs.iterator());
104
	}
105

  
106
	/**
107
	 * {@inheritDoc}
108
	 *
109
	 * @see eu.dnetlib.functionality.index.IndexCollection#deleteIndex(java.lang.String)
110
	 */
111
	@Override
112
	public boolean deleteIndex(final String dsId) throws IndexServiceException {
113
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
114
		return doDelete(IndexFieldUtility.DS_ID + " : \"" + dsId + "\"");
115
	}
116

  
117
	/**
118
	 * {@inheritDoc}
119
	 *
120
	 * @see eu.dnetlib.functionality.index.IndexCollection#deleteByQuery(java.lang.String, java.lang.String)
121
	 */
122
	@Override
123
	public boolean deleteByQuery(final String query, final String dsId) throws IndexServiceException {
124
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
125
		if (StringUtils.isBlank(dsId)) return doDelete(query);
126
		return doDelete(query + " AND " + IndexFieldUtility.DS_ID + " : \"" + dsId + "\"");
127
	}
128

  
129
	/**
130
	 * Do delete.
131
	 *
132
	 * @param query
133
	 *            the query
134
	 * @return true, if do delete
135
	 * @throws IndexServiceException
136
	 *             the index service exception
137
	 */
138
	protected boolean doDelete(final String query) throws IndexServiceException {
139
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
140
		try {
141
			log.debug("delete by query: " + query);
142
			return server.deleteByQuery(query).getStatus() == STATUS_INDEX_OK;
143
		} catch (final Exception e) {
144
			throw new IndexServiceException("unable to run delete by query: " + query, e);
145
		}
146
	}
147

  
148
	/**
149
	 * {@inheritDoc}
150
	 *
151
	 * @see eu.dnetlib.functionality.index.IndexCollection#commit()
152
	 */
153
	@Override
154
	public boolean commit() throws IndexServiceException {
155
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
156
		try {
157
			log.info("performing commit");
158
			final UpdateResponse rsp = server.commit();
159
			log.info(String.format("commit completed in %s, status %s", rsp.getElapsedTime(), rsp.getStatus()));
160
			return rsp.getStatus() == STATUS_INDEX_OK;
161
		} catch (final Throwable e) {
162
			throw new IndexServiceException("unable to perform index commit", e);
163
		}
164
	}
165

  
166
	@Override
167
	public void shutdown() {
168
		server.shutdown();
169
		shutdown = true;
170
	}
171

  
172
	public boolean isShutdown() {
173
		return shutdown;
174
	}
175

  
176
	public void setShutdown(final boolean shutdown) {
177
		this.shutdown = shutdown;
178
	}
179

  
180
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/applicationContext-index-solr-service.properties
1
service.solr.index.staticConfigurationClasspath= eu/dnetlib/functionality/index/conf/files/**
2
service.solr.index.jsonConfiguration= \
3
	{"id":"solr",\
4
	"address":"localhost:9983",\
5
	"port":"8983",\
6
	"webContext":"solr",\
7
	"numShards":"1",\
8
	"replicationFactor":"1",\
9
    "maxShardsPerNode":"2",\
10
	"host":"localhost",\
11
	"feedingShutdownTolerance":"30000",\
12
	"feedingBufferFlushThreshold":"1000",\
13
	"feedingSimulationMode":"false",\
14
	"luceneMatchVersion":"4.9",\
15
	"serverLibPath":"../../../../contrib/extraction/lib",\
16
	"filterCacheSize":"512","filterCacheInitialSize":"512",\
17
	"queryCacheSize":"512","queryCacheInitialSize":"512",\
18
	"documentCacheSize":"512","documentCacheInitialSize":"512",\
19
	"ramBufferSizeMB":"960","mergeFactor":"40",\
20
	"autosoftcommit":"-1","autocommit":"15000",\
21
	"termIndexInterval":"1024","maxIndexingThreads":"8",\
22
	"queryResultWindowSize":"20","queryResultMaxDocCached":"200"} 
23

  
24
service.index.solr.returnEmptyFields		= true
25
service.index.solr.schema.textfieldtype		= text_common
26
service.index.solr.nh.mdformat.enable		= true
27
service.index.solr.highlight.enable			= true
28
service.index.solr.rank.enable				= true
29
service.index.feed.repeatDelay				= 20000
30
services.mapreduce.index.solr.feed.sim.mode	= false
31
service.index.solr.document.factory			= eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory
32
service.index.solr.schema.template			= classpath:/eu/dnetlib/functionality/index/conf/schemaTemplate.xslt
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/admin-extra.menu-top.html
1
<!-- admin-extra.menu-top.html -->
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/update-script.js
1
/*
2
  This is a basic skeleton JavaScript update processor.
3

  
4
  In order for this to be executed, it must be properly wired into solrconfig.xml; by default it is commented out in
5
  the example solrconfig.xml and must be uncommented to be enabled.
6

  
7
  See http://wiki.apache.org/solr/ScriptUpdateProcessor for more details.
8
*/
9

  
10
function processAdd(cmd) {
11

  
12
  doc = cmd.solrDoc;  // org.apache.solr.common.SolrInputDocument
13
  id = doc.getFieldValue("id");
14
  logger.info("update-script#processAdd: id=" + id);
15

  
16
// Set a field value:
17
//  doc.setField("foo_s", "whatever");
18

  
19
// Get a configuration parameter:
20
//  config_param = params.get('config_param');  // "params" only exists if processor configured with <lst name="params">
21

  
22
// Get a request parameter:
23
// some_param = req.getParams().get("some_param")
24

  
25
// Add a field of field names that match a pattern:
26
//   - Potentially useful to determine the fields/attributes represented in a result set, via faceting on field_name_ss
27
//  field_names = doc.getFieldNames().toArray();
28
//  for(i=0; i < field_names.length; i++) {
29
//    field_name = field_names[i];
30
//    if (/attr_.*/.test(field_name)) { doc.addField("attribute_ss", field_names[i]); }
31
//  }
32

  
33
}
34

  
35
function processDelete(cmd) {
36
  // no-op
37
}
38

  
39
function processMergeIndexes(cmd) {
40
  // no-op
41
}
42

  
43
function processCommit(cmd) {
44
  // no-op
45
}
46

  
47
function processRollback(cmd) {
48
  // no-op
49
}
50

  
51
function finish() {
52
  // no-op
53
}
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/admin-extra.menu-bottom.html
1
<!-- admin-extra.menu-bottom.html -->
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/protwords.txt
1
# The ASF licenses this file to You under the Apache License, Version 2.0
2
# (the "License"); you may not use this file except in compliance with
3
# the License.  You may obtain a copy of the License at
4
#
5
#     http://www.apache.org/licenses/LICENSE-2.0
6
#
7
# Unless required by applicable law or agreed to in writing, software
8
# distributed under the License is distributed on an "AS IS" BASIS,
9
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
# See the License for the specific language governing permissions and
11
# limitations under the License.
12

  
13
#-----------------------------------------------------------------------
14
# Use a protected word file to protect against the stemmer reducing two
15
# unrelated words to the same base word.
16

  
17
# Some non-words that normally won't be encountered,
18
# just to test that they won't be stemmed.
19
dontstems
20
zwhacky
21

  
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/stopwords.txt
1
# Licensed to the Apache Software Foundation (ASF) under one or more
2
# contributor license agreements.  See the NOTICE file distributed with
3
# this work for additional information regarding copyright ownership.
4
# The ASF licenses this file to You under the Apache License, Version 2.0
5
# (the "License"); you may not use this file except in compliance with
6
# the License.  You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
modules/dnet-index-solr-service/tags/dnet-index-solr-service-2.4.3/src/main/resources/eu/dnetlib/functionality/index/conf/files/elevate.xml
1
<?xml version="1.0" encoding="UTF-8" ?>
2
<!--
3
 Licensed to the Apache Software Foundation (ASF) under one or more
4
 contributor license agreements.  See the NOTICE file distributed with
5
 this work for additional information regarding copyright ownership.
6
 The ASF licenses this file to You under the Apache License, Version 2.0
7
 (the "License"); you may not use this file except in compliance with
8
 the License.  You may obtain a copy of the License at
9

  
10
     http://www.apache.org/licenses/LICENSE-2.0
11

  
12
 Unless required by applicable law or agreed to in writing, software
13
 distributed under the License is distributed on an "AS IS" BASIS,
14
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
 See the License for the specific language governing permissions and
16
 limitations under the License.
17
-->
18

  
19
<!-- If this file is found in the config directory, it will only be
20
     loaded once at startup.  If it is found in Solr's data
21
     directory, it will be re-loaded every commit.
22

  
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff