Revision 51138
Added by Claudio Atzori about 6 years ago
modules/dnet-fs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java | ||
---|---|---|
6 | 6 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile; |
7 | 7 |
import eu.dnetlib.data.objectstore.rmi.Protocols; |
8 | 8 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
9 |
import org.apache.commons.lang.StringUtils; |
|
9 |
import org.apache.commons.lang3.StringUtils;
|
|
10 | 10 |
import org.apache.commons.logging.Log; |
11 | 11 |
import org.apache.commons.logging.LogFactory; |
12 | 12 |
|
modules/dnet-fs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStore.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.data.objectstore.filesystem; |
2 | 2 |
|
3 |
import com.google.common.base.Function; |
|
3 |
import java.io.ByteArrayInputStream; |
|
4 |
import java.io.IOException; |
|
5 |
import java.nio.file.FileSystems; |
|
6 |
import java.nio.file.Files; |
|
7 |
import java.nio.file.Path; |
|
8 |
import java.util.regex.Pattern; |
|
9 |
|
|
4 | 10 |
import com.google.common.collect.Iterables; |
5 | 11 |
import com.google.common.collect.Lists; |
6 | 12 |
import com.google.gson.Gson; |
... | ... | |
17 | 23 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; |
18 | 24 |
import eu.dnetlib.enabling.resultset.ResultSetListener; |
19 | 25 |
import eu.dnetlib.miscutils.collections.Pair; |
20 |
import org.apache.commons.lang.StringUtils; |
|
26 |
import org.apache.commons.lang3.StringUtils;
|
|
21 | 27 |
import org.apache.commons.logging.Log; |
22 | 28 |
import org.apache.commons.logging.LogFactory; |
23 | 29 |
import org.bson.conversions.Bson; |
24 | 30 |
|
25 |
import java.io.ByteArrayInputStream; |
|
26 |
import java.io.IOException; |
|
27 |
import java.nio.file.FileSystems; |
|
28 |
import java.nio.file.Files; |
|
29 |
import java.nio.file.Path; |
|
30 |
import java.util.regex.Pattern; |
|
31 |
|
|
32 | 31 |
/** |
33 | 32 |
* The Class FileSystemObjectStore. |
34 | 33 |
* |
... | ... | |
122 | 121 |
throw new ObjectStoreServiceException("Error can't feed objects because the folder " + baseDirPath + " does not exist"); |
123 | 122 |
|
124 | 123 |
int addedCounter = 0; |
124 |
int nulls = 0; |
|
125 | 125 |
for (ObjectStoreRecord record : records) { |
126 | 126 |
String url = feedObject(record); |
127 | 127 |
if (StringUtils.isNotBlank(url)) { |
128 | 128 |
addedCounter++; |
129 |
} else { |
|
130 |
nulls++; |
|
129 | 131 |
} |
130 | 132 |
} |
133 |
if (nulls > 0) { |
|
134 |
log.warn(String.format("Found %s null records", nulls)); |
|
135 |
} |
|
131 | 136 |
return addedCounter; |
132 | 137 |
} |
133 | 138 |
|
... | ... | |
140 | 145 |
*/ |
141 | 146 |
@Override |
142 | 147 |
public int feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException { |
143 |
Iterable<ObjectStoreRecord> it = Iterables.transform(records, new Function<MetadataObjectRecord, ObjectStoreRecord>() { |
|
144 |
@Override |
|
145 |
public ObjectStoreRecord apply(final MetadataObjectRecord metadataObjectRecord) { |
|
146 |
ObjectStoreRecord r = new ObjectStoreRecord(); |
|
147 |
r.setInputStream(new ByteArrayInputStream(metadataObjectRecord.getRecord().getBytes())); |
|
148 |
ObjectStoreFile fileMetadata = new ObjectStoreFile(); |
|
149 |
fileMetadata.setObjectID(metadataObjectRecord.getId()); |
|
150 |
fileMetadata.setMimeType(metadataObjectRecord.getMime()); |
|
151 |
r.setFileMetadata(fileMetadata); |
|
152 |
return r; |
|
153 |
} |
|
148 |
Iterable<ObjectStoreRecord> it = Iterables.transform(records, mor -> { |
|
149 |
ObjectStoreRecord r = new ObjectStoreRecord(); |
|
150 |
r.setInputStream(new ByteArrayInputStream(mor.getRecord().getBytes())); |
|
151 |
ObjectStoreFile fileMetadata = new ObjectStoreFile(); |
|
152 |
fileMetadata.setObjectID(mor.getId()); |
|
153 |
fileMetadata.setMimeType(mor.getMime()); |
|
154 |
r.setFileMetadata(fileMetadata); |
|
155 |
return r; |
|
154 | 156 |
}); |
155 | 157 |
return feed(it, incremental); |
156 | 158 |
} |
... | ... | |
217 | 219 |
} |
218 | 220 |
} |
219 | 221 |
} |
220 |
log.warn("Record for object store is null"); |
|
221 | 222 |
return null; |
222 | 223 |
} |
223 | 224 |
|
modules/dnet-fs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java | ||
---|---|---|
1 |
/** |
|
2 |
* |
|
3 |
*/ |
|
4 | 1 |
package eu.dnetlib.data.objectstore.filesystem; |
5 | 2 |
|
6 |
import eu.dnetlib.miscutils.collections.Pair; |
|
7 |
import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions; |
|
8 |
import org.apache.commons.lang.StringUtils; |
|
9 |
import org.apache.commons.logging.Log; |
|
10 |
import org.apache.commons.logging.LogFactory; |
|
11 |
|
|
12 | 3 |
import java.io.FileInputStream; |
13 | 4 |
import java.io.IOException; |
14 | 5 |
import java.io.InputStream; |
15 | 6 |
import java.nio.file.*; |
16 | 7 |
import java.nio.file.attribute.BasicFileAttributes; |
17 | 8 |
|
9 |
import eu.dnetlib.miscutils.collections.Pair; |
|
10 |
import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions; |
|
11 |
import org.apache.commons.lang3.StringUtils; |
|
12 |
import org.apache.commons.logging.Log; |
|
13 |
import org.apache.commons.logging.LogFactory; |
|
14 |
|
|
18 | 15 |
/** |
19 | 16 |
* The Class FileSystemUtility. |
20 | 17 |
* |
modules/dnet-fs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java | ||
---|---|---|
6 | 6 |
import com.mongodb.client.MongoDatabase; |
7 | 7 |
import com.mongodb.client.model.IndexOptions; |
8 | 8 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao; |
9 |
import org.apache.commons.lang.StringUtils; |
|
9 |
import org.apache.commons.lang3.StringUtils;
|
|
10 | 10 |
import org.apache.commons.logging.Log; |
11 | 11 |
import org.apache.commons.logging.LogFactory; |
12 | 12 |
import org.springframework.beans.factory.annotation.Autowired; |
modules/dnet-fs-objectstore/trunk/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreDao.java | ||
---|---|---|
1 |
/** |
|
2 |
* |
|
3 |
*/ |
|
4 | 1 |
package eu.dnetlib.data.objectstore.filesystem; |
5 | 2 |
|
6 | 3 |
import java.io.IOException; |
... | ... | |
22 | 19 |
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; |
23 | 20 |
import eu.dnetlib.miscutils.collections.MappedCollection; |
24 | 21 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
25 |
import org.apache.commons.lang.StringUtils; |
|
22 |
import org.apache.commons.lang3.StringUtils;
|
|
26 | 23 |
import org.apache.commons.logging.Log; |
27 | 24 |
import org.apache.commons.logging.LogFactory; |
28 | 25 |
import org.bson.conversions.Bson; |
Also available in: Unified diff
using apache commons lang3