Project

General

Profile

« Previous | Next » 

Revision 49516

do not index invisible records

View differences:

modules/dnet-mapreduce-jobs/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/index/PrepareFeedMapper.java
68 68
		final Oaf oaf = mergeUpdates(value, context, type, keyDecoder);
69 69

  
70 70
		if (isValid(oaf)) {
71
			if (!isInvisible(oaf)) {
72
				if (!deletedByInference(oaf) || entityConfigTable.includeDuplicates(type)) {
73
					emit(new String(keyIn.copyBytes()), context, oaf);
71 74

  
72
			if (!deletedByInference(oaf) || entityConfigTable.includeDuplicates(type)) {
73
				emit(new String(keyIn.copyBytes()), context, oaf);
75
					incrementCounter(context, Kind.entity.toString(), getEntityType(oaf, type), 1);
74 76

  
75
				incrementCounter(context, Kind.entity.toString(), getEntityType(oaf, type), 1);
77
					final Collection<LinkDescriptor> lds = entityConfigTable.getDescriptors(type);
78
					if (lds.isEmpty()) {
79
						context.getCounter(type.name(), "missing link descriptor").increment(1);
80
					}
81
					for (final LinkDescriptor ld : lds) {
76 82

  
77
				final Collection<LinkDescriptor> lds = entityConfigTable.getDescriptors(type);
78
				if (lds.isEmpty()) {
79
					context.getCounter(type.name(), "missing link descriptor").increment(1);
80
				}
81
				for (final LinkDescriptor ld : lds) {
83
						final Map<byte[], byte[]> columnMap = value.getFamilyMap(Bytes.toBytes(ld.getRelDescriptor().getIt()));
82 84

  
83
					final Map<byte[], byte[]> columnMap = value.getFamilyMap(Bytes.toBytes(ld.getRelDescriptor().getIt()));
84

  
85
					if (hasData(columnMap)) {
86
						emitRelationship(oaf.getEntity(), context, columnMap, ld);
87
						incrementCounter(context, type.toString(), ld.getRelDescriptor().getIt(), columnMap.size());
88
					} // else {
85
						if (hasData(columnMap)) {
86
							emitRelationship(oaf.getEntity(), context, columnMap, ld);
87
							incrementCounter(context, type.toString(), ld.getRelDescriptor().getIt(), columnMap.size());
88
						} // else {
89 89
						// incrementCounter(context, type.toString(), ld.getRelDescriptor().getIt() + "_empty", 1);
90 90
						// }
91
					}
92
				} else {
93
					incrementCounter(context, "deleted by inference", type.toString(), 1);
91 94
				}
92 95
			} else {
93
				incrementCounter(context, "deleted by inference", type.toString(), 1);
96
				incrementCounter(context, "invisible", type.toString(), 1);
94 97
			}
95 98
		} else {
96 99
			incrementCounter(context, "missing body (map)", type.toString(), 1);
......
147 150
		}
148 151
	}
149 152

  
150
	private String patchTargetId(final Type target, final String id) {
151
		return id.replaceFirst("^.*\\|", target.getNumber() + "|");
152
	}
153

  
154 153
	private Oaf.Builder merge(final Oaf.Builder builder, final Oaf prototype) {
155 154
		return builder.setDataInfo(prototype.getDataInfo()).setLastupdatetimestamp(prototype.getLastupdatetimestamp());
156 155
	}
......
167 166
		return (oaf != null) && oaf.isInitialized();
168 167
	}
169 168

  
169
	private boolean isInvisible(final Oaf oaf) {
170
		return oaf.getDataInfo().hasInvisible() ? oaf.getDataInfo().getInvisible() : false;
171
	}
172

  
170 173
	private boolean deletedByInference(final Oaf oaf) {
171 174
		return oaf.getDataInfo().getDeletedbyinference();
172 175
	}

Also available in: Unified diff