Revision 41729
Added by Eri Katsari about 8 years ago
| modules/dnet-openaire-stats-workflow/branch/src/main/resources/eu/dnetlib/iis/core/javamapreduce/stats/job.properties | ||
|---|---|---|
| 1 | 1 |
# Fri Jan 29 12:09:02 EET 2016 |
| 2 | 2 |
#------CNR ----- |
| 3 |
oozieServiceLoc=http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie |
|
| 4 |
nameNode=hdfs://nmis-hadoop-cluster |
|
| 5 |
jobTracker=nmis-hadoop-jt |
|
| 3 |
#oozieServiceLoc=http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie
|
|
| 4 |
#nameNode=hdfs://nmis-hadoop-cluster
|
|
| 5 |
#jobTracker=nmis-hadoop-jt
|
|
| 6 | 6 |
#-----ICM IIS------ |
| 7 | 7 |
#oozieServiceLoc=http://gateway.hadoop.iis.openaire.eu:11000/oozie |
| 8 | 8 |
#nameNode=hdfs://iis-cluster-nn |
| 9 | 9 |
#jobTracker=iis-cluster-jt |
| 10 | 10 |
#-----DM ICM ------ |
| 11 |
#nameNode=hdfs://dm-cluster-nn
|
|
| 12 |
#jobTracker=dm-cluster-jt
|
|
| 13 |
#oozieServiceLoc=http://oozie.hadoop.dm.openaire.eu:11000/oozie
|
|
| 11 |
nameNode=hdfs://dm-cluster-nn |
|
| 12 |
jobTracker=dm-cluster-jt |
|
| 13 |
oozieServiceLoc=http://oozie.hadoop.dm.openaire.eu:11000/oozie |
|
| 14 | 14 |
#queueName=default |
| 15 | 15 |
Stats_Column_Families=result |
| 16 |
Stats_Hbase_Source_Table=db_openaireplus_services |
|
| 16 |
Stats_Hbase_Source_Table=db_openaireplus_services_beta
|
|
| 17 | 17 |
Stats_db_Driver=org.postgresql.Driver |
| 18 | 18 |
Stats_db_Pass=vereniki |
| 19 |
Stats_db_Url=jdbc:postgresql://194.177.192.118:5432/stats |
|
| 19 |
Stats_db_Url=jdbc:postgresql://194.177.192.118:5432/stats_new
|
|
| 20 | 20 |
Stats_db_User=postgres |
| 21 | 21 |
Stats_usageDB_Driver=org.sqlite.JDBC |
| 22 | 22 |
Stats_usageDB_url=jdbc:sqlite:/tmp/piwikOpenAireportal.db |
| ... | ... | |
| 35 | 35 |
#jobTracker=dm-cluster-jt |
| 36 | 36 |
#nameNode=hdfs://dm-cluster-nn |
| 37 | 37 |
numReducers=3 |
| 38 |
oozie.wf.application.path=hdfs://nmis-hadoop-cluster/user/eri.katsari/proj/oozie_app |
|
| 38 |
#oozie.wf.application.path=hdfs://nmis-hadoop-cluster/user/eri.katsari/proj/oozie_app |
|
| 39 |
oozie.wf.application.path=hdfs://dm-cluster-nn/user/eri.katsari/proj/oozie_app |
|
| 39 | 40 |
#oozieServiceLoc=http://oozie.hadoop.dm.openaire.eu:11000/oozie |
| 40 | 41 |
out1=datasource |
| 41 | 42 |
out2=project |
| modules/dnet-openaire-stats-workflow/branch/src/main/resources/eu/dnetlib/iis/core/javamapreduce/stats/oozie_app/workflow.xml | ||
|---|---|---|
| 140 | 140 |
<name>mapreduce.multipleoutputs</name> |
| 141 | 141 |
|
| 142 | 142 |
<value> |
| 143 |
${out1} ${out2} ${out3} ${out4} ${out5} ${out6} ${out7} ${out8} ${out9} ${out10} ${out11}
|
|
| 144 |
${out12} ${out13} ${out14} ${out15} ${out16} ${out17} ${out18} ${out19} ${out20} ${out21}
|
|
| 145 |
${out22} ${out23} ${out24} ${out25} ${out26} ${out27} ${out28} ${out29} ${out30} ${out31}
|
|
| 146 |
${out32} ${out33} ${out34} ${out35} ${out36}${out37} ${out38}
|
|
| 143 |
${out1} ${out2} ${out3} ${out4} ${out5} ${out6} ${out7} ${out8} ${out9} ${out10} ${out11} ${out12} ${out13} ${out14} ${out15} ${out16} ${out17} ${out18} ${out19} ${out20} ${out21} ${out22} ${out23} ${out24} ${out25} ${out26} ${out27} ${out28} ${out29} ${out30} ${out31} ${out32} ${out33} ${out34} ${out35} ${out36}${out37} ${out38}
|
|
| 147 | 144 |
</value> |
| 148 | 145 |
|
| 149 | 146 |
</property> |
| modules/dnet-openaire-stats/branches/full_export/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/utils/Serializer.java | ||
|---|---|---|
| 416 | 416 |
|
| 417 | 417 |
// number?? |
| 418 | 418 |
|
| 419 |
|
|
| 419 | 420 |
buff += getStringField("1");
|
| 420 | 421 |
return buff; |
| 421 | 422 |
} |
| modules/dnet-openaire-stats/branches/full_export/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/mapreduce/StatsMapper.java | ||
|---|---|---|
| 7 | 7 |
import eu.dnetlib.data.mapreduce.hbase.statsExport.utils.Serializer; |
| 8 | 8 |
import eu.dnetlib.data.mapreduce.util.OafRowKeyDecoder; |
| 9 | 9 |
import eu.dnetlib.data.mapreduce.util.UpdateMerger; |
| 10 |
import eu.dnetlib.data.proto.OafProtos; |
|
| 11 | 10 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
| 12 | 11 |
import eu.dnetlib.data.proto.OafProtos.OafRel; |
| 13 | 12 |
import eu.dnetlib.data.proto.OafProtos.OafRelOrBuilder; |
| 14 |
import eu.dnetlib.data.proto.ProjectProtos; |
|
| 15 |
import eu.dnetlib.data.proto.RelTypeProtos; |
|
| 16 | 13 |
import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; |
| 17 |
import eu.dnetlib.data.proto.TypeProtos; |
|
| 18 | 14 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
| 19 | 15 |
import org.apache.hadoop.hbase.client.Result; |
| 20 | 16 |
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; |
| 21 | 17 |
import org.apache.hadoop.hbase.mapreduce.TableMapper; |
| 22 | 18 |
import org.apache.hadoop.hbase.util.Bytes; |
| 23 | 19 |
import org.apache.hadoop.io.Text; |
| 24 |
import org.apache.hadoop.mapreduce.Mapper.Context; |
|
| 25 | 20 |
import org.apache.log4j.Logger; |
| 26 | 21 |
|
| 27 |
import javax.management.relation.Relation; |
|
| 28 | 22 |
import java.io.ByteArrayInputStream; |
| 29 | 23 |
import java.io.IOException; |
| 30 | 24 |
import java.io.InputStream; |
Also available in: Unified diff
test for production with beta table