s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate GuessTypesOfColumnCommand to Jackson

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-20 15:21:09 +00:00
parent 772ee1ee59
commit 59175daf38
4 changed files with 74 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -43,6 +43,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -51,23 +52,24 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconType;
import com.google.refine.model.Row;
import com.google.refine.model.recon.StandardReconConfig.ReconResult;
import com.google.refine.util.ParsingUtilities;
public class GuessTypesOfColumnCommand extends Command {
protected static class TypesResponse {
@JsonProperty("code")
protected String code;
@ -135,7 +137,7 @@ public class GuessTypesOfColumnCommand extends Command {
* @throws JSONException, IOException
*/
protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl)
throws JSONException, IOException {
throws IOException {
Map<String, TypeGroup> map = new HashMap<String, TypeGroup>();
int cellIndex = column.getCellIndex();
@ -193,48 +195,35 @@ public class GuessTypesOfColumnCommand extends Command {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
for (int i = 0; i < samples.size(); i++) {
String key = "q" + i;
if (!o.has(key)) {
Iterator<JsonNode> iterator = o.iterator();
while (iterator.hasNext()) {
JsonNode o2 = iterator.next();
if (!(o2.has("result") && o2.get("result") instanceof ArrayNode)) {
continue;
}
JSONObject o2 = o.getJSONObject(key);
if (!(o2.has("result"))) {
continue;
}
JSONArray results = o2.getJSONArray("result");
int count = results.length();
ArrayNode results = (ArrayNode) o2.get("result");
List<ReconResult> reconResults = ParsingUtilities.mapper.convertValue(results, new TypeReference<List<ReconResult>>() {});
int count = reconResults.size();
for (int j = 0; j < count; j++) {
JSONObject result = results.getJSONObject(j);
ReconResult result = reconResults.get(j);
double score = 1.0 / (1 + j); // score by each result's rank
JSONArray types = result.getJSONArray("type");
int typeCount = types.length();
List<ReconType> types = result.types;
int typeCount = types.size();
for (int t = 0; t < typeCount; t++) {
Object type = types.get(t);
String typeID;
String typeName;
if (type instanceof String) {
typeID = typeName = (String) type;
} else {
typeID = ((JSONObject) type).getString("id");
typeName = ((JSONObject) type).getString("name");
}
ReconType type = types.get(t);
double score2 = score * (typeCount - t) / typeCount;
if (map.containsKey(typeID)) {
TypeGroup tg = map.get(typeID);
if (map.containsKey(type.id)) {
TypeGroup tg = map.get(type.id);
tg.score += score2;
tg.count++;
} else {
map.put(typeID, new TypeGroup(typeID, typeName, score2));
map.put(type.id, new TypeGroup(type.id, type.name, score2));
}
}
}

10
main/src/com/google/refine/model/ReconType.java
View File

@ -57,8 +57,14 @@ public class ReconType {
String id,
@JsonProperty("name")
String name) {
this.id = id;
this.name = name;
this.id = id;
this.name = name;
}
@JsonCreator
public ReconType(String id) {
this.id = id;
this.name = null;
}
static public ReconType load(String json) throws IOException {

50
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -286,17 +286,33 @@ public class StandardReconConfig extends ReconConfig {
}
}
protected class ReconResult {
public class ReconResult {
@JsonProperty("name")
protected String name;
public String name;
@JsonProperty("id")
protected String id;
public String id;
@JsonProperty("types")
protected String[] types = new String[0];
public List<ReconType> types = Collections.emptyList();
@JsonProperty("score")
protected double score;
public double score;
@JsonProperty("match")
protected boolean match = false;
public boolean match = false;
@JsonIgnore
public ReconCandidate toCandidate() {
String[] bareTypes = new String[types.size()];
for(int i = 0; i != types.size(); i++) {
bareTypes[i] = types.get(i).id;
}
ReconCandidate result = new ReconCandidate(
id,
name,
bareTypes,
score
);
return result;
}
}
@Override
@ -455,25 +471,21 @@ public class StandardReconConfig extends ReconConfig {
protected Recon createReconServiceResults(String text, ArrayNode resultsList, long historyEntryID) throws IOException {
Recon recon = new Recon(historyEntryID, identifierSpace, schemaSpace);
List<ReconResult> results = ParsingUtilities.mapper.readValue(resultsList.toString(), new TypeReference<List<ReconResult>>() {});
List<ReconResult> results = ParsingUtilities.mapper.convertValue(resultsList, new TypeReference<List<ReconResult>>() {});
int length = results.size();
int count = 0;
for (int i = 0; i < length; i++) {
ReconResult result = results.get(i);
ReconCandidate candidate = new ReconCandidate(
result.id,
result.name,
result.types,
result.score
);
if (autoMatch && i == 0 && result.match) {
recon.match = candidate;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
}
ReconCandidate candidate = result.toCandidate();
if (autoMatch && i == 0 && result.match) {
recon.match = candidate;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
}
recon.addCandidate(candidate);
count++;

13
main/tests/server/src/com/google/refine/tests/model/ReconTypeTest.java
View File

@ -1,6 +1,8 @@
package com.google.refine.tests.model;
import org.json.JSONException;
import java.io.IOException;
import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.refine.model.ReconType;
@ -8,9 +10,16 @@ import com.google.refine.tests.util.TestUtils;
public class ReconTypeTest {
@Test
public void serializeReconType() throws JSONException, Exception {
public void serializeReconType() throws IOException {
String json = "{\"id\":\"Q7540126\",\"name\":\"headquarters\"}";
ReconType rt = ReconType.load(json);
TestUtils.isSerializedTo(rt, json);
}
@Test
public void deserializeFromString() throws IOException {
// reconciliation services can return lists of types as bare lists of strings
ReconType rt = ReconType.load("\"Q7540126\"");
Assert.assertEquals(rt.id, "Q7540126");
}
}