Revision 53068
Added by Claudio Atzori about 6 years ago
| modules/dnet-mapreduce-jobs/branches/beta/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataimport/CrossRefToActions.java | ||
|---|---|---|
| 27 | 27 |
|
| 28 | 28 |
public static AtomicAction generateActionsFromDump(final JsonObject rootElement, ActionFactory factory, final String setName, final Agent agent, boolean invisible) {
|
| 29 | 29 |
|
| 30 |
|
|
| 31 |
|
|
| 32 | 30 |
//Create OAF Proto |
| 33 | 31 |
final Oaf.Builder oaf = Oaf.newBuilder(); |
| 34 | 32 |
//Add Data Info |
| ... | ... | |
| 66 | 64 |
); |
| 67 | 65 |
} |
| 68 | 66 |
|
| 69 |
|
|
| 70 | 67 |
//Adding identifier |
| 71 | 68 |
final String objIdentifier = getStringValue(rootElement, "objIdentifier"); |
| 72 | 69 |
final String nsPrefix = getStringValue(rootElement,"datasourcePrefix"); |
| ... | ... | |
| 169 | 166 |
|
| 170 | 167 |
//Adding date |
| 171 | 168 |
String date = getStringValue(rootElement,"issued"); |
| 172 |
if (date.length()==4) |
|
| 173 |
date +="-01-01"; |
|
| 169 |
if (date.length() == 4) {
|
|
| 170 |
date += "-01-01"; |
|
| 171 |
} |
|
| 174 | 172 |
|
| 175 |
metadata.setDateofacceptance(StringField.newBuilder().setValue(date).build()); |
|
| 173 |
if (isValidDate(date)) {
|
|
| 174 |
metadata.setDateofacceptance(StringField.newBuilder().setValue(date).build()); |
|
| 175 |
} |
|
| 176 | 176 |
|
| 177 | 177 |
//Adding description |
| 178 | 178 |
String description=null; |
| ... | ... | |
| 211 | 211 |
entity.setResult(result.build()); |
| 212 | 212 |
oaf.setEntity(entity.build()); |
| 213 | 213 |
|
| 214 |
|
|
| 215 |
|
|
| 216 |
|
|
| 217 | 214 |
return factory.createAtomicAction(setName, agent,oaf.getEntity().getId(), "result", "body",oaf.build().toByteArray()); |
| 218 | 215 |
} |
| 219 | 216 |
|
| 220 |
|
|
| 221 | 217 |
private static boolean hasJSONArrayField(final JsonObject root, final String key) {
|
| 222 | 218 |
return root.has(key) && root.get(key).isJsonArray(); |
| 223 | 219 |
} |
| 224 | 220 |
|
| 225 |
|
|
| 226 | 221 |
public static List<Author> createAuthors(final JsonObject root) {
|
| 227 | 222 |
|
| 228 | 223 |
if (root.has("author") && root.get("author").isJsonArray()) {
|
| ... | ... | |
| 257 | 252 |
|
| 258 | 253 |
} |
| 259 | 254 |
return null; |
| 260 |
|
|
| 261 |
|
|
| 262 | 255 |
} |
| 263 | 256 |
|
| 264 |
|
|
| 265 |
|
|
| 266 |
|
|
| 267 | 257 |
} |
| modules/dnet-mapreduce-jobs/branches/beta/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataimport/DumpToActionsUtility.java | ||
|---|---|---|
| 32 | 32 |
} |
| 33 | 33 |
return new ArrayList<>(); |
| 34 | 34 |
} |
| 35 |
|
|
| 36 |
public static boolean isValidDate(final String date) {
|
|
| 37 |
return date.matches("\\d{4}-\\d{2}-\\d{2}");
|
|
| 38 |
} |
|
| 39 |
|
|
| 35 | 40 |
public static FieldTypeProtos.StructuredProperty getPid(final JsonObject localIdentifier, final Map<String, ScholExplorerConfiguration> conf) {
|
| 36 | 41 |
final String pidType = getStringValue(localIdentifier, "type"); |
| 37 | 42 |
final ScholExplorerConfiguration configuration = conf.get(pidType); |
| modules/dnet-mapreduce-jobs/branches/beta/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataimport/ScholixToActions.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mapreduce.hbase.dataimport; |
| 2 | 2 |
|
| 3 |
import java.text.SimpleDateFormat; |
|
| 4 |
import java.util.ArrayList; |
|
| 5 |
import java.util.List; |
|
| 6 |
import java.util.Map; |
|
| 7 |
import java.util.Objects; |
|
| 8 |
|
|
| 9 |
import com.google.common.collect.Lists; |
|
| 10 | 3 |
import com.google.gson.JsonArray; |
| 11 | 4 |
import com.google.gson.JsonElement; |
| 12 | 5 |
import com.google.gson.JsonObject; |
| 13 |
import com.googlecode.protobuf.format.JsonFormat; |
|
| 14 | 6 |
import eu.dnetlib.actionmanager.actions.ActionFactory; |
| 15 | 7 |
import eu.dnetlib.actionmanager.actions.AtomicAction; |
| 16 | 8 |
import eu.dnetlib.actionmanager.common.Agent; |
| ... | ... | |
| 20 | 12 |
import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions; |
| 21 | 13 |
import org.apache.commons.lang3.StringUtils; |
| 22 | 14 |
|
| 15 |
import java.util.ArrayList; |
|
| 16 |
import java.util.List; |
|
| 17 |
import java.util.Map; |
|
| 18 |
import java.util.Objects; |
|
| 19 |
|
|
| 20 |
import static eu.dnetlib.data.mapreduce.hbase.dataimport.DumpToActionsUtility.*; |
|
| 23 | 21 |
import static eu.dnetlib.data.proto.FieldTypeProtos.*; |
| 24 | 22 |
import static eu.dnetlib.data.proto.KindProtos.Kind; |
| 25 | 23 |
import static eu.dnetlib.data.proto.OafProtos.*; |
| ... | ... | |
| 27 | 25 |
import static eu.dnetlib.data.proto.ResultProtos.Result.*; |
| 28 | 26 |
import static eu.dnetlib.data.proto.ResultResultProtos.ResultResult; |
| 29 | 27 |
import static eu.dnetlib.data.proto.TypeProtos.Type; |
| 30 |
import static eu.dnetlib.data.mapreduce.hbase.dataimport.DumpToActionsUtility.*; |
|
| 31 | 28 |
|
| 32 | 29 |
public class ScholixToActions {
|
| 33 | 30 |
|
| ... | ... | |
| 328 | 325 |
return result; |
| 329 | 326 |
} |
| 330 | 327 |
|
| 331 |
private static boolean isValidDate(final String date) {
|
|
| 332 |
return date.matches("\\d{4}-\\d{2}-\\d{2}");
|
|
| 333 |
} |
|
| 334 |
|
|
| 335 |
|
|
| 336 |
|
|
| 337 |
|
|
| 338 |
|
|
| 339 | 328 |
} |
Also available in: Unified diff
rule out invalid dates also on CrossRefToActions