/ - Diff - D-Net - D-Net project tracking tool

« Previous | Next »

Revision 29336

Added by Eri Katsari about 10 years ago

     	private Logger log = Logger.getLogger(this.getClass());
     	private String NULL_STRING;
     	private String NULL_NUM;
     	private static String ENCLOSED;
     	public String serialize(Oaf oaf) {
-...
     	private String buildresultProject(OafRel oaf) {
     		String buff = new String();
     		buff +=  getStringField(oaf.getTarget());
     		buff += getStringField(oaf.getTarget());
     		// TODO is declared as int!!!
     		buff += getYearDifferenceInteger(oaf.getResultProject().getOutcome().getRelMetadata().getEnddate(), oaf.getResultProject().getOutcome().getRelMetadata().getStartdate());
-...
     			String hostedBy = null;
     			hostedBy = instance.getHostedby().getKey();
     			buffs.add((getStringField(hostedBy)));
+    		}
     		rels.put("resultDatasource", buffs);
-...
     		Datasource d = data.getDatasource();
     		Metadata metadata = d.getMetadata();
     		String full_id =getStringField(data.getId());
     		String full_id = getStringField(data.getId());
     		buff += full_id;
     		buff += full_id;
-...
     		eu.dnetlib.data.proto.OrganizationProtos.Organization.Metadata metadata = organization.getMetadata();
     		// `organization_datasources`,
     		String full_id =getStringField(data.getId());
     		String full_id = getStringField(data.getId());
     		buff += full_id;
     		buff += full_id;
     		// organization_projects
     		buff += full_id;
     		buff += full_id;
     		// `name`,
     		buff += getStringField(metadata.getLegalname().getValue());
     		// `country`,
-...
     		if (metadata.getCountry().getClassname().equals("UNITED KINGDOM"))
+    		{
     			buff +=
     					buff += getStringField("United Kingdom");
     			buff += buff += getStringField("United Kingdom");
     		} else if (metadata.getCountry().getClassname().equals("GREECE")) {
     			buff += getStringField ("Greece");
     			buff += getStringField("Greece");
     		} else
+    		{
-...
     		eu.dnetlib.data.proto.ResultProtos.Result.Metadata metadata = result.getMetadata();
     		// result_topics/
     		String full_id =getStringField(data.getId());
     		String full_id = getStringField(data.getId());
     		buff += full_id;
     		// result_languages
-...
     		// number??
     		buff += getStringField("1");
     		return buff;
     		// TODO check if valid or empty protobuff
     		//TODO do it in all protos?
     		if (isValid(buff, full_id)) {
     			return buff;
+    		}
     		else {
     			return null;
+    		}
+    	}
     	private boolean isValid(String buff, String id) {
     		return buff.endsWith(id);
+    	}
     	private String getBestLicense(Result result) {
     		Qualifier bestLicense = null;
     		LicenseComparator lc = new LicenseComparator();
-...
     		eu.dnetlib.data.proto.ProjectProtos.Project.Metadata metadata = project.getMetadata();
     		// project_organizations
     		String full_id =getStringField(data.getId());
     		buff +=  full_id;
     		String full_id = getStringField(data.getId());
     		buff += full_id;
     		// project_results
     		buff += full_id;
-...
     		split = split[1].split(",");
     		funding_level = split[0].replaceAll(".*:\"", "");
     		funding_level = funding_level.replaceFirst("\"", "");
     		funding_level = funding_level.replaceFirst(ENCLOSED, "");
     		funding_level = funding_level.trim();
     		return funding_level;
-...
     			int Enddate = Integer.parseInt(split[0]);
     			int diff = Enddate - Startdate;
     			return "\"" + diff + "\"" + DELIM;
     			return ENCLOSED + diff + ENCLOSED + DELIM;
+    		}
     		return "\"" + NULL_NUM + "\"" + DELIM;
     		return ENCLOSED + NULL_NUM + ENCLOSED + DELIM;
+    	}
     	private String getYearInt(String data) {
     		if (data == null || data.isEmpty()) {
     			return "\"" + NULL_NUM + "\"" + DELIM;
     			return ENCLOSED + NULL_NUM + ENCLOSED + DELIM;
+    		}
     		String[] split = data.split("-");
     		if (split != null) {
     			return "\"" + split[0] + "\"" + DELIM;
     			return ENCLOSED + split[0] + ENCLOSED + DELIM;
     		} else {
     			return "\"" + NULL_NUM + "\"" + DELIM;
     			return ENCLOSED + NULL_NUM + ENCLOSED + DELIM;
+    		}
+    	}
     	private String getStringField(String data) {
     		if (data == null || data.isEmpty() || data.equals("")) {
     			return "\"" + NULL_STRING + "\"" + DELIM;
     			return ENCLOSED + NULL_STRING + ENCLOSED + DELIM;
     		} else {
     		return clean(data) + DELIM;
     			return clean(data) + DELIM;
+    		}
+    	}
     	private String getNumericField(String data) {
     		if (data == null || data.isEmpty()) {
     			return "\"" + NULL_NUM + "\"" + DELIM;
     			return ENCLOSED + NULL_NUM + ENCLOSED + DELIM;
     		} else {
     			return "\"" + data + "\"" + DELIM;
     			return ENCLOSED + data + ENCLOSED + DELIM;
+    		}
+    	}
-...
     			value = value.replaceFirst(".*\\|", "");
     			value = value.replaceAll("\n", "");
     			value = value.trim();
+    		}
     		if (!value.contains("\"")) {
     			return "\"" + value + "\"";
     		if (!value.contains(ENCLOSED)) {
     			return ENCLOSED + value + ENCLOSED;
     		} else {
     			return value;
+    		}
-...
     		NULL_NUM = nULL_NUM;
+    	}
     	public static String getENCLOSED() {
     		return ENCLOSED;
+    	}
     	public void setENCLOSED(String eNCLOSED) {
     		ENCLOSED = eNCLOSED;
+    	}
+    }

     		serializer.setDELIM(context.getConfiguration().get("stats.delim"));
     		serializer.setNULL_NUM(context.getConfiguration().get("stats.nullNum"));
     		serializer.setNULL_STRING(context.getConfiguration().get("stats.nullString"));
     		serializer.setENCLOSED(context.getConfiguration().get("stats.enclChar"));
+    	}
     	@Override

     import org.apache.hadoop.fs.RemoteIterator;
     import org.apache.log4j.Logger;
     import org.apache.sqoop.Sqoop;
     import org.junit.experimental.runners.Enclosed;
     import com.google.common.collect.ArrayListMultimap;
     import com.google.common.collect.Multimap;
-...
     	private boolean batch = true;
     	private boolean verbose = true;
     	private String tableMapConf;
     	private String enclosed;
     	/**
     	 * Driver for the Sqoop tool. Calls the Sqoop Client for each <input file,
-...
     			"--connect", connectionUrl, "--table", table.getKey(),
     			"--export-dir", table.getValue(),
     			 "--input-fields-terminated-by", delim,
     			 "--input-enclosed-by", "\"",
     			 "--verbose", "--username", dbUser, "--password", dbPass, "--driver", "org.postgresql.Driver", "--batch", "--mapreduce-job-name", "Sqoop Stats Import Job for " + table.getKey(), "--m", sqoopReducersCount };
     			"--input-fields-terminated-by", delim, "--input-enclosed-by", enclosed,
     			"--verbose", "--username", dbUser, "--password", dbPass, "--driver", "org.postgresql.Driver", "--batch", "--mapreduce-job-name", "Sqoop Stats Import Job for " + table.getKey(), "--m", sqoopReducersCount };
     			int ret = Sqoop.runTool(str);
     			if (ret != 0) {
-...
     				if (split[0].equals(name)) {
     					tables.put((String) e.getValue(), filename);
     					log.info("    match   " +  e.getValue()  + "  " + filename);
     					log.info("    match   " + e.getValue() + "  " + filename);
+    				}
+    			}
+    		}
     		long startTime = System.currentTimeMillis();
     		log.info("    match   " +  tables.entries());
     		log.info("    match   " + tables.entries());
     		try {
     			this.run(tables);
-...
     		this.tableMapConf = tableMapConf;
+    	}
     	public String getEnclosed() {
     		return enclosed;
+    	}
     	public void setEnclosed(String enclosed) {
     		this.enclosed = enclosed;
+    	}
+    }

Also available in: Unified diff

Project

General

Profile

D-Net

Revision 29336

Added by Eri Katsari about 10 years ago