Revision 33173
Added by Eri Katsari almost 10 years ago
| modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/utils/ContextExporter.java | ||
|---|---|---|
| 71 | 71 |
|
| 72 | 72 |
writeData(context, outputPath + "context"); |
| 73 | 73 |
writeData(category, outputPath + "category"); |
| 74 |
writeData(concept, outputPath + "concept");
|
|
| 74 |
writeData(concept, outputPath + "concept"); |
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 | 77 |
private void readFromBuffer(String contextMap) throws Exception {
|
| ... | ... | |
| 102 | 102 |
|
| 103 | 103 |
private void processData(String data) throws Exception {
|
| 104 | 104 |
try {
|
| 105 |
|
|
| 105 | 106 |
String[] split = data.split("COPY");
|
| 107 |
|
|
| 108 |
|
|
| 106 | 109 |
context += split[0]; |
| 110 |
|
|
| 107 | 111 |
split[1] = split[1].replaceFirst("\n", "");
|
| 108 |
category += split[1].substring(0, split[1].lastIndexOf("\n") - 1)+"\n";
|
|
| 109 |
split[2] = split[2].replaceFirst("\n", "");
|
|
| 110 |
concept += split[2].substring(0, split[2].lastIndexOf("\n") - 1)+"\n";
|
|
| 112 |
if(!category.isEmpty()) |
|
| 111 | 113 |
|
| 114 |
{ category = category.substring(0, category.lastIndexOf("\n") - 1);
|
|
| 115 |
} |
|
| 116 |
category += split[1]+"\n"; |
|
| 112 | 117 |
|
| 118 |
if(concept.isEmpty()) |
|
| 119 |
|
|
| 120 |
{
|
|
| 121 |
split[2] = split[2].replaceFirst("\n", "");
|
|
| 122 |
|
|
| 123 |
} concept += split[2]+"\n"; |
|
| 124 |
|
|
| 113 | 125 |
} catch (Exception e) {
|
| 114 | 126 |
String msg = " Unable to create file with context, " + "concept and category values in output path " + outputPath + ". Reason: "; |
| 115 | 127 |
log.error(msg); |
| ... | ... | |
| 122 | 134 |
|
| 123 | 135 |
FSDataOutputStream fin = null; |
| 124 | 136 |
try {
|
| 125 |
data = data.replaceFirst("\n", "");
|
|
| 126 |
data = data.substring(0, data.lastIndexOf("\n") - 1);
|
|
| 127 |
log.info(" ++++++++++++++" + destination + "++++++++++++++");
|
|
| 128 |
log.info(" Writing data: " + data);
|
|
| 129 |
|
|
| 137 |
|
|
| 138 |
data = data.replaceAll("\n\n","");
|
|
| 139 |
data = data.substring(0, data.lastIndexOf("\n") );
|
|
| 140 |
if(data.endsWith("\n"))
|
|
| 141 |
{
|
|
| 142 |
data = data.substring(0, data.lastIndexOf("\n") );
|
|
| 143 |
} |
|
| 144 |
log.info("***********************Writing data:***********************\n" + data); log.info("*********************** data:***********************\n" );
|
|
| 130 | 145 |
FileSystem fs = FileSystem.get(new Configuration()); |
| 131 | 146 |
fin = fs.create(new Path(destination), true); |
| 132 | 147 |
|
Also available in: Unified diff
updated xslt for concept to ommit newlines that were causing an issue with FET concept entries