dnet40/modules/dnet-openaire-lodinterlinking/branches/cacheOptimized/src/test/java/BuildTest.java @ 46462
1 |
import com.lambdaworks.redis.RedisClient; |
---|---|
2 |
import com.lambdaworks.redis.RedisConnection; |
3 |
import com.lambdaworks.redis.RedisURI; |
4 |
import eu.dnetlib.data.mapreduce.hbase.lodExport.build.BlockReducer; |
5 |
import eu.dnetlib.data.mapreduce.hbase.lodExport.build.SourceBuildMapper; |
6 |
import eu.dnetlib.data.mapreduce.hbase.lodExport.build.TargetBuildMapper; |
7 |
import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.caching.RedisUtils; |
8 |
import org.apache.hadoop.conf.Configuration; |
9 |
import org.apache.hadoop.io.LongWritable; |
10 |
import org.apache.hadoop.io.Text; |
11 |
import org.apache.hadoop.mrunit.mapreduce.MapDriver; |
12 |
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; |
13 |
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; |
14 |
import org.junit.Before; |
15 |
import org.junit.Test; |
16 |
import redis.clients.jedis.Connection; |
17 |
|
18 |
import java.io.IOException; |
19 |
import java.util.ArrayList; |
20 |
|
21 |
public class BuildTest { |
22 |
MapDriver<LongWritable, Text, Text, Text> sourceMapDriver; |
23 |
ReduceDriver<Text, Text, Text, Text> reduceDriver; |
24 |
MapReduceDriver<LongWritable, Text, Text, Text, Text, Text> mapReduceDriver; |
25 |
private MapDriver<LongWritable, Text, Text, Text> targetMapDriver; |
26 |
|
27 |
@Before
|
28 |
public void setUp() { |
29 |
SourceBuildMapper sourceBuildMapper = new SourceBuildMapper(); |
30 |
TargetBuildMapper targetBuildMapper = new TargetBuildMapper(); |
31 |
BlockReducer blockReducer = new BlockReducer(); |
32 |
|
33 |
Configuration configuration = new Configuration(); |
34 |
String sourceMappings = "{\"type\":\"result\", \"fields\":[\"<http://purl.org/dc/terms/identifier>\",\"<http://www.eurocris.org/ontologies/cerif/1.3#name>\",\"<http://lod.openaire.eu/vocab/year>\"]}"; |
35 |
configuration.set("lod.sourceMappings", sourceMappings); |
36 |
|
37 |
String stopwords = "a,able,about,above,abst,accordance,according,accordingly,across,act,actually,added,adj,affected,affecting,affects,after,afterwards,again,against,ah,all,almost,alone,along,already,also,although,always,am,among,amongst,an,and,announce,another,any,anybody,anyhow,anymore,anyone,anything,anyway,anyways,anywhere,apparently,approximately,are,aren,arent,arise,around,as,aside,ask,asking,at,auth,available,away,awfully,b,back,be,became,because,become,becomes,becoming,been,before,beforehand,begin,beginning,beginnings,begins,behind,being,believe,below,beside,besides,between,beyond,biol,both,brief,briefly,but,by,c,ca,came,can,cannot,cant,cause,causes,certain,certainly,co,com,come,comes,contain,containing,contains,could,couldnt,d,date,did,didnt,different,do,does,doesnt,doing,done,dont,down,downwards,due,during,e,each,ed,edu,effect,eg,eight,eighty,either,else,elsewhere,end,ending,enough,especially,et,et-al,etc,even,ever,every,everybody,everyone,everything,everywhere,ex,except,f,far,few,ff,fifth,first,five,fix,followed,following,follows,for,former,formerly,forth,found,four,from,further,furthermore,g,gave,get,gets,getting,give,given,gives,giving,go,goes,gone,got,gotten,h,had,happens,hardly,has,hasnt,have,havent,having,he,hed,hence,her,here,hereafter,hereby,herein,heres,hereupon,hers,herself,hes,hi,hid,him,himself,his,hither,home,how,howbeit,however,hundred,i,id,ie,if,ill,im,immediate,immediately,importance,important,in,inc,indeed,index,information,instead,into,invention,inward,is,isnt,it,itd,itll,its,itself,ive,j,just,k,keep, keeps,kept,kg,km,know,known,knows,l,largely,last,lately,later,latter,latterly,least,less,lest,let,lets,like,liked,likely,line,little,ll,look,looking,looks,ltd,m,made,mainly,make,makes,many,may,maybe,me,mean,means,meantime,meanwhile,merely,mg,might,million,miss,ml,more,moreover,most,mostly,mr,mrs,much,mug,must,my,myself,n,na,name,namely,nay,nd,near,nearly,necessarily,necessary,need,needs,neither,never,nevertheless,new,next,nine,ninety,no,nobody,non,none,nonetheless,noone,nor,normally,nos,not,noted,nothing,now,nowhere,o,obtain,obtained,obviously,of,off,often,oh,ok,okay,old,omitted,on,once,one,ones,only,onto,or,ord,other,others,otherwise,ought,our,ours,ourselves,out,outside,over,overall,owing,own,p,page,pages,part,particular,particularly,past,per,perhaps,placed,please,plus,poorly,possible,possibly,potentially,pp,predominantly,present,previously,primarily,probably,promptly,proud,provides,put,q,que,quickly,quite,qv,r,ran,rather,rd,re,readily,really,recent,recently,ref,refs,regarding,regardless,regards,related,relatively,research,respectively,resulted,resulting,results,right,run,s,said,same,saw,say,saying,says,sec,section,see,seeing,seem,seemed,seeming,seems,seen,self,selves,sent,seven,several,shall,she,shed,shell,shes,should,shouldnt,show,showed,shown,showns,shows,significant,significantly,similar,similarly,since,six,slightly,so,some,somebody,somehow,someone,somethan,something,sometime,sometimes,somewhat,somewhere,soon,sorry,specifically,specified,specify,specifying,still,stop,strongly,sub,substantially,successfully,such,sufficiently,suggest,sup,sure,t,take,taken,taking,tell,tends,th,than,thank,thanks,thanx,that,thatll,thats,thatve,the,their,theirs,them,themselves,then,thence,there,thereafter,thereby,thered,therefore,therein,therell,thereof,therere,theres,thereto,thereupon,thereve,these,they,theyd,theyll,theyre,theyve,think,this,those,thou,though,thoughh,thousand,throug,through,throughout,thru,thus,til,tip,to,together,too,took,toward,towards,tried,tries,truly,try,trying,ts,twice,two,u,un,under,unfortunately,unless,unlike,unlikely,until,unto,up,upon,ups,us,use,used,useful,usefully,usefulness,uses,using,usually,v,value,various,ve,very,via,viz,vol,vols,vs,w,want,wants,was,wasnt,way,we,wed,welcome,well,went,were,werent,weve,what,whatever,whatll,whats,when,whence,whenever,where,whereafter,whereas,whereby,wherein,wheres,whereupon,wherever,whether,which,while,whim,whither,who,whod,whoever,whole,wholl,whom,whomever,whos,whose,why,widely,willing,wish,with,within,without,wont,words,world,would,wouldnt,www,x,y,yes,yet,you,youd,youll,your,youre,yours,yourself,yourselves,youve,z,zero\n"; |
38 |
configuration.set("lod.stopwords", stopwords); |
39 |
|
40 |
configuration.set("lod.redisHost", "83.212.96.39"); |
41 |
configuration.set("lod.redisPort", "6379"); |
42 |
configuration.set("lod.redisPass", "TOXxhc0!53-A"); |
43 |
|
44 |
|
45 |
String targetMappings = "{\"type\":\"publications\",\"fields\":[\"<http://purl.org/dc/terms/issued>\",\"<http://www.w3.org/2000/01/rdf-schema#label>\",\"<http://purl.org/dc/terms/identifier>\"]}"; |
46 |
configuration.set("lod.targetMappings", targetMappings); |
47 |
|
48 |
sourceMapDriver = MapDriver.newMapDriver(sourceBuildMapper).withConfiguration(configuration); |
49 |
targetMapDriver = MapDriver.newMapDriver(targetBuildMapper).withConfiguration(configuration); |
50 |
|
51 |
reduceDriver = ReduceDriver.newReduceDriver(blockReducer).withConfiguration(configuration); |
52 |
mapReduceDriver = MapReduceDriver.newMapReduceDriver(sourceBuildMapper, blockReducer).withConfiguration(configuration); |
53 |
}
|
54 |
|
55 |
@Test
|
56 |
public void testSourceMapper() throws IOException { |
57 |
sourceMapDriver.withInput(new LongWritable(1), new Text("<http://lod.openaire.eu/data/result/acm_________::008169b761b014b88105a9ed96bb0b4c>\t<http://purl.org/dc/terms/identifier>\t\"1570486\"\t.\t<http://lod.openaire.eu/data/result/acm_________::008169b761b014b88105a9ed96bb0b4c>\t<http://purl.org/dc/terms/identifier>\t\"acm_________::008169b761b014b88105a9ed96bb0b4c\"\t.\t<http://lod.openaire.eu/data/result/acm_________::008169b761b014b88105a9ed96bb0b4c>\t<http://purl.org/dc/terms/identifier>\t\"10.1145/1570433.1570486\"\t.\t<http://lod.openaire.eu/data/result/acm_________::008169b761b014b88105a9ed96bb0b4c>\t<http://www.eurocris.org/ontologies/cerif/1.3#name>\t\"MundoMonkey\"\t.\t<http://lod.openaire.eu/data/result/acm_________::008169b761b014b88105a9ed96bb0b4c>\t<http://lod.openaire.eu/vocab/year>\t\"2009\"\t.\t\t")); |
58 |
sourceMapDriver.withOutput(new Text("lala"), new Text("lalal")); |
59 |
sourceMapDriver.run(); |
60 |
}
|
61 |
|
62 |
|
63 |
@Test
|
64 |
public void testTargetMapper() throws IOException { |
65 |
|
66 |
targetMapDriver.withInput(new LongWritable(1), new Text("<http://dblp.l3s.de/d2r/resource/publications/books/acm/0082477>\t<http://purl.org/dc/terms/issued>\t\"1992\"\t.\t<http://dblp.l3s.de/d2r/resource/publications/books/acm/0082477>\t<http://www.w3.org/2000/01/rdf-schema#label>\t\"The no-nonsense guide to computing careers.\"\t.\t")); |
67 |
targetMapDriver.run(); |
68 |
}
|
69 |
|
70 |
@Test
|
71 |
public void testReducer() throws IOException { |
72 |
ArrayList list = new ArrayList(); |
73 |
list.add(new Text("source_<http://dblp.l3s.de/d2r/resource/publications/books/acm/0082477>")); |
74 |
list.add(new Text("target_<http://dblp.l3s.de/d2r/resource/publications/books/acm/00824www7>")); |
75 |
reduceDriver.withInput(new Text("lalal"), list); |
76 |
reduceDriver.withOutput(new Text("text"), new Text("tetx")); |
77 |
reduceDriver.run(); |
78 |
}
|
79 |
|
80 |
@Test
|
81 |
public void redisTest() { |
82 |
RedisURI redisUri = RedisURI.Builder.redis("83.212.96.39") |
83 |
.withPassword("TOXxhc0!53-A") |
84 |
.withPort(6379) |
85 |
.build(); |
86 |
RedisClient client = RedisClient.create(redisUri); |
87 |
RedisConnection connection = client.connect(); |
88 |
System.out.println(connection.keys("*")); |
89 |
connection.close(); |
90 |
}
|
91 |
|
92 |
}
|
93 |
|
94 |
|