Revision 45504
Added by Michele Artini over 7 years ago
| BulkResourceImporter.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.enabling.tools; |
| 2 | 2 |
|
| 3 |
import eu.dnetlib.enabling.is.registry.schema.OpaqueResourceValidator; |
|
| 4 |
import eu.dnetlib.enabling.is.registry.schema.ValidationException; |
|
| 5 |
import eu.dnetlib.enabling.is.store.ISStore; |
|
| 6 |
import eu.dnetlib.enabling.is.store.ISStoreException; |
|
| 7 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 8 |
import eu.dnetlib.rmi.enabling.ISRegistryException; |
|
| 3 |
import java.io.IOException; |
|
| 4 |
import java.util.Date; |
|
| 9 | 5 |
|
| 6 |
import org.apache.commons.io.FilenameUtils; |
|
| 7 |
import org.apache.commons.io.IOUtils; |
|
| 8 |
import org.apache.commons.logging.Log; |
|
| 9 |
import org.apache.commons.logging.LogFactory; |
|
| 10 |
import org.dom4j.Document; |
|
| 11 |
import org.dom4j.io.SAXReader; |
|
| 10 | 12 |
import org.springframework.beans.factory.annotation.Autowired; |
| 11 |
import org.springframework.beans.factory.annotation.Required; |
|
| 13 |
import org.springframework.beans.factory.annotation.Value; |
|
| 14 |
import org.springframework.context.ResourceLoaderAware; |
|
| 15 |
import org.springframework.core.io.Resource; |
|
| 16 |
import org.springframework.core.io.ResourceLoader; |
|
| 17 |
import org.springframework.core.io.support.ResourcePatternUtils; |
|
| 12 | 18 |
|
| 13 |
/** |
|
| 14 |
* This class implements a bulk resource import, i.e. importing stuff straight into the store, bypassing checks and policies imposed by the |
|
| 15 |
* registry service. |
|
| 16 |
* |
|
| 17 |
* TODO: move from registry based to store based. |
|
| 18 |
* |
|
| 19 |
* @author marko |
|
| 20 |
* |
|
| 21 |
*/ |
|
| 22 |
public class BulkResourceImporter {
|
|
| 19 |
import eu.dnetlib.DnetConstants; |
|
| 20 |
import eu.dnetlib.enabling.is.DnetInformationServiceException; |
|
| 21 |
import eu.dnetlib.enabling.is.registry.schema.ProfileValidator; |
|
| 22 |
import eu.dnetlib.enabling.is.store.ISStore; |
|
| 23 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
| 23 | 24 |
|
| 24 |
/** |
|
| 25 |
* xquery utils, needed to map resources with the xmldb collection names. |
|
| 26 |
*/ |
|
| 27 |
private XQueryUtils xqueryUtils; |
|
| 25 |
public class BulkResourceImporter implements ResourceLoaderAware {
|
|
| 28 | 26 |
|
| 29 |
/** |
|
| 30 |
* service locator. |
|
| 31 |
*/ |
|
| 32 |
private UniqueServiceLocator serviceLocator; |
|
| 27 |
static final Log log = LogFactory.getLog(BulkResourceImporter.class); |
|
| 33 | 28 |
|
| 34 |
/** |
|
| 35 |
* resource validator. |
|
| 36 |
*/ |
|
| 37 |
private OpaqueResourceValidator resourceValidator; |
|
| 29 |
private static boolean initialized = false; |
|
| 38 | 30 |
|
| 39 |
/** |
|
| 40 |
* set to false to skip validation. |
|
| 41 |
*/ |
|
| 42 |
private boolean validating = true; |
|
| 31 |
@Autowired |
|
| 32 |
private XQueryUtils xqueryUtils; |
|
| 43 | 33 |
|
| 44 |
/** |
|
| 45 |
* builk loading enabled. |
|
| 46 |
*/ |
|
| 47 |
private boolean enabled = true; |
|
| 34 |
@Autowired |
|
| 35 |
private ProfileValidator profileValidator; |
|
| 48 | 36 |
|
| 49 | 37 |
@Autowired |
| 50 | 38 |
private ISStore isStore; |
| 51 | 39 |
|
| 52 |
/** |
|
| 53 |
* register a resource bypassing the checks. |
|
| 54 |
* |
|
| 55 |
* @param resource |
|
| 56 |
* a resource |
|
| 57 |
* @throws ISRegistryException |
|
| 58 |
* could happen |
|
| 59 |
*/ |
|
| 60 |
public void importResource(final OpaqueResource resource) throws ISRegistryException {
|
|
| 40 |
@Value("TODO")
|
|
| 41 |
private String bootstrapDir; |
|
| 42 |
|
|
| 43 |
private ResourceLoader resourceLoader; |
|
| 44 |
|
|
| 45 |
public void startImport() throws DnetInformationServiceException {
|
|
| 46 |
if (isInitialized()) { return; }
|
|
| 47 |
|
|
| 48 |
log.info("Initializing store with some profiles and schemas ...");
|
|
| 49 |
|
|
| 50 |
log.info("Loading profiles and schemas from " + bootstrapDir);
|
|
| 51 |
|
|
| 61 | 52 |
try {
|
| 62 |
if (validating) {
|
|
| 63 |
resourceValidator.validate(resource); |
|
| 53 |
|
|
| 54 |
final long start = System.currentTimeMillis(); |
|
| 55 |
|
|
| 56 |
for (final Resource res : ResourcePatternUtils.getResourcePatternResolver(getResourceLoader()).getResources(bootstrapDir + "/**/*.xsd")) {
|
|
| 57 |
registerSchema(res); |
|
| 64 | 58 |
} |
| 65 |
isStore.insertXML(xqueryUtils.getFileName(resource), xqueryUtils.getCollectionAbsPath(resource), resource.asString()); |
|
| 66 |
} catch (final ISStoreException e) {
|
|
| 67 |
throw new ISRegistryException(e); |
|
| 68 |
} catch (final ValidationException e) {
|
|
| 69 |
throw new ISRegistryException(e); |
|
| 59 |
|
|
| 60 |
for (final Resource res : ResourcePatternUtils.getResourcePatternResolver(getResourceLoader()).getResources(bootstrapDir + "/**/*.xml")) {
|
|
| 61 |
registerProfile(res); |
|
| 62 |
} |
|
| 63 |
|
|
| 64 |
log.info("bulk registration finished in: " + ((System.currentTimeMillis() - start) / 1000) + "s");
|
|
| 65 |
|
|
| 66 |
} catch (final IOException e) {
|
|
| 67 |
throw new DnetInformationServiceException(e); |
|
| 68 |
} finally {
|
|
| 69 |
log.info("INITIALIZED");
|
|
| 70 |
setInitialized(true); |
|
| 70 | 71 |
} |
| 71 | 72 |
} |
| 72 | 73 |
|
| 73 |
public XQueryUtils getXqueryUtils() {
|
|
| 74 |
return xqueryUtils; |
|
| 74 |
private void registerSchema(final Resource res) throws DnetInformationServiceException {
|
|
| 75 |
final String resourceType = FilenameUtils.getBaseName(res.getFilename()); |
|
| 76 |
try {
|
|
| 77 |
isStore.insertXML(resourceType, xqueryUtils.getRootCollection() + DnetConstants.RESOURCE_TYPES, IOUtils.toString(res.getInputStream())); |
|
| 78 |
log.info("new schema imported: " + resourceType);
|
|
| 79 |
} catch (final Throwable e) {
|
|
| 80 |
log.error("error importing schema " + res.getFilename(), e);
|
|
| 81 |
throw new DnetInformationServiceException(e); |
|
| 82 |
} |
|
| 75 | 83 |
} |
| 76 | 84 |
|
| 77 |
@Required |
|
| 78 |
public void setXqueryUtils(final XQueryUtils xqueryUtils) {
|
|
| 79 |
this.xqueryUtils = xqueryUtils; |
|
| 80 |
} |
|
| 85 |
private void registerProfile(final Resource res) throws DnetInformationServiceException {
|
|
| 86 |
log.debug("registering profile");
|
|
| 81 | 87 |
|
| 82 |
@Required |
|
| 83 |
public void setResourceValidator(final OpaqueResourceValidator resourceValidator) {
|
|
| 84 |
this.resourceValidator = resourceValidator; |
|
| 85 |
} |
|
| 88 |
try {
|
|
| 89 |
final Document doc = (new SAXReader()).read(res.getInputStream()); |
|
| 86 | 90 |
|
| 87 |
public OpaqueResourceValidator getResourceValidator() {
|
|
| 88 |
return resourceValidator; |
|
| 89 |
} |
|
| 91 |
final String id = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
|
|
| 92 |
final String type = doc.valueOf("//RESOURCE_TYPE/@value");
|
|
| 90 | 93 |
|
| 91 |
public void setValidating(final boolean validating) {
|
|
| 92 |
this.validating = validating;
|
|
| 93 |
}
|
|
| 94 |
final String coll = xqueryUtils.getCollectionPath(doc);
|
|
| 95 |
final String fileName = xqueryUtils.getFileName(id);
|
|
| 96 |
final String date = new DateUtils(new Date()).getDateAsISO8601String();
|
|
| 94 | 97 |
|
| 95 |
public boolean isValidating() {
|
|
| 96 |
return validating; |
|
| 98 |
doc.selectSingleNode("//DATE_OF_CREATION/@value").setText(date);
|
|
| 99 |
|
|
| 100 |
final String finalDoc = doc.asXML(); |
|
| 101 |
|
|
| 102 |
if (profileValidator.isValid(type, finalDoc)) {
|
|
| 103 |
isStore.insertXML(fileName, xqueryUtils.getRootCollection() + coll, finalDoc); |
|
| 104 |
} else {
|
|
| 105 |
throw new DnetInformationServiceException("Invalid profile");
|
|
| 106 |
} |
|
| 107 |
log.info("new profile imported: " + xqueryUtils.getRootCollection() + coll + fileName);
|
|
| 108 |
} catch (final Throwable e) {
|
|
| 109 |
log.error("error importing profile " + res.getFilename());
|
|
| 110 |
throw new DnetInformationServiceException(e); |
|
| 111 |
} |
|
| 97 | 112 |
} |
| 98 | 113 |
|
| 99 |
public boolean isEnabled() {
|
|
| 100 |
return enabled;
|
|
| 114 |
public boolean isInitialized() {
|
|
| 115 |
return initialized;
|
|
| 101 | 116 |
} |
| 102 | 117 |
|
| 103 |
public void setEnabled(final boolean enabled) {
|
|
| 104 |
this.enabled = enabled;
|
|
| 118 |
public void setInitialized(final boolean initialized) {
|
|
| 119 |
BulkResourceImporter.initialized = initialized;
|
|
| 105 | 120 |
} |
| 106 | 121 |
|
| 107 |
public UniqueServiceLocator getServiceLocator() {
|
|
| 108 |
return serviceLocator;
|
|
| 122 |
public ResourceLoader getResourceLoader() {
|
|
| 123 |
return resourceLoader;
|
|
| 109 | 124 |
} |
| 110 | 125 |
|
| 111 |
@Required
|
|
| 112 |
public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
|
|
| 113 |
this.serviceLocator = serviceLocator;
|
|
| 126 |
@Override
|
|
| 127 |
public void setResourceLoader(final ResourceLoader resourceLoader) {
|
|
| 128 |
this.resourceLoader = resourceLoader;
|
|
| 114 | 129 |
} |
| 115 | 130 |
|
| 116 | 131 |
} |
Also available in: Unified diff