s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate GuessTypesOfColumnCommand to Jackson

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-20 15:21:09 +00:00
parent 772ee1ee59
commit 59175daf38
4 changed files with 74 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -43,6 +43,7 @@ import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -51,23 +52,24 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Column; import com.google.refine.model.Column;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.ReconType;
import com.google.refine.model.Row; import com.google.refine.model.Row;
import com.google.refine.model.recon.StandardReconConfig.ReconResult;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
public class GuessTypesOfColumnCommand extends Command { public class GuessTypesOfColumnCommand extends Command {
protected static class TypesResponse { protected static class TypesResponse {
@JsonProperty("code") @JsonProperty("code")
protected String code; protected String code;
@ -135,7 +137,7 @@ public class GuessTypesOfColumnCommand extends Command {
* @throws JSONException, IOException * @throws JSONException, IOException
*/ */
protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl) protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl)
throws JSONException, IOException { throws IOException {
Map<String, TypeGroup> map = new HashMap<String, TypeGroup>(); Map<String, TypeGroup> map = new HashMap<String, TypeGroup>();
int cellIndex = column.getCellIndex(); int cellIndex = column.getCellIndex();
@ -193,48 +195,35 @@ public class GuessTypesOfColumnCommand extends Command {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
for (int i = 0; i < samples.size(); i++) { Iterator<JsonNode> iterator = o.iterator();
String key = "q" + i; while (iterator.hasNext()) {
if (!o.has(key)) { JsonNode o2 = iterator.next();
if (!(o2.has("result") && o2.get("result") instanceof ArrayNode)) {
continue; continue;
} }
JSONObject o2 = o.getJSONObject(key); ArrayNode results = (ArrayNode) o2.get("result");
if (!(o2.has("result"))) { List<ReconResult> reconResults = ParsingUtilities.mapper.convertValue(results, new TypeReference<List<ReconResult>>() {});
continue; int count = reconResults.size();
}
JSONArray results = o2.getJSONArray("result");
int count = results.length();
for (int j = 0; j < count; j++) { for (int j = 0; j < count; j++) {
JSONObject result = results.getJSONObject(j); ReconResult result = reconResults.get(j);
double score = 1.0 / (1 + j); // score by each result's rank double score = 1.0 / (1 + j); // score by each result's rank
JSONArray types = result.getJSONArray("type"); List<ReconType> types = result.types;
int typeCount = types.length(); int typeCount = types.size();
for (int t = 0; t < typeCount; t++) { for (int t = 0; t < typeCount; t++) {
Object type = types.get(t); ReconType type = types.get(t);
String typeID;
String typeName;
if (type instanceof String) {
typeID = typeName = (String) type;
} else {
typeID = ((JSONObject) type).getString("id");
typeName = ((JSONObject) type).getString("name");
}
double score2 = score * (typeCount - t) / typeCount; double score2 = score * (typeCount - t) / typeCount;
if (map.containsKey(typeID)) { if (map.containsKey(type.id)) {
TypeGroup tg = map.get(typeID); TypeGroup tg = map.get(type.id);
tg.score += score2; tg.score += score2;
tg.count++; tg.count++;
} else { } else {
map.put(typeID, new TypeGroup(typeID, typeName, score2)); map.put(type.id, new TypeGroup(type.id, type.name, score2));
} }
} }
} }

10
main/src/com/google/refine/model/ReconType.java
View File

@ -57,8 +57,14 @@ public class ReconType {
String id, String id,
@JsonProperty("name") @JsonProperty("name")
String name) { String name) {
this.id = id; this.id = id;
this.name = name; this.name = name;
}
@JsonCreator
public ReconType(String id) {
this.id = id;
this.name = null;
} }
static public ReconType load(String json) throws IOException { static public ReconType load(String json) throws IOException {

50
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -286,17 +286,33 @@ public class StandardReconConfig extends ReconConfig {
} }
} }
protected class ReconResult { public class ReconResult {
@JsonProperty("name") @JsonProperty("name")
protected String name; public String name;
@JsonProperty("id") @JsonProperty("id")
protected String id; public String id;
@JsonProperty("types") @JsonProperty("types")
protected String[] types = new String[0]; public List<ReconType> types = Collections.emptyList();
@JsonProperty("score") @JsonProperty("score")
protected double score; public double score;
@JsonProperty("match") @JsonProperty("match")
protected boolean match = false; public boolean match = false;
@JsonIgnore
public ReconCandidate toCandidate() {
String[] bareTypes = new String[types.size()];
for(int i = 0; i != types.size(); i++) {
bareTypes[i] = types.get(i).id;
}
ReconCandidate result = new ReconCandidate(
id,
name,
bareTypes,
score
);
return result;
}
} }
@Override @Override
@ -455,25 +471,21 @@ public class StandardReconConfig extends ReconConfig {
protected Recon createReconServiceResults(String text, ArrayNode resultsList, long historyEntryID) throws IOException { protected Recon createReconServiceResults(String text, ArrayNode resultsList, long historyEntryID) throws IOException {
Recon recon = new Recon(historyEntryID, identifierSpace, schemaSpace); Recon recon = new Recon(historyEntryID, identifierSpace, schemaSpace);
List<ReconResult> results = ParsingUtilities.mapper.readValue(resultsList.toString(), new TypeReference<List<ReconResult>>() {}); List<ReconResult> results = ParsingUtilities.mapper.convertValue(resultsList, new TypeReference<List<ReconResult>>() {});
int length = results.size(); int length = results.size();
int count = 0; int count = 0;
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
ReconResult result = results.get(i); ReconResult result = results.get(i);
ReconCandidate candidate = new ReconCandidate(
result.id,
result.name,
result.types,
result.score
);
if (autoMatch && i == 0 && result.match) { ReconCandidate candidate = result.toCandidate();
recon.match = candidate;
recon.matchRank = 0; if (autoMatch && i == 0 && result.match) {
recon.judgment = Judgment.Matched; recon.match = candidate;
recon.judgmentAction = "auto"; recon.matchRank = 0;
} recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
}
recon.addCandidate(candidate); recon.addCandidate(candidate);
count++; count++;

13
main/tests/server/src/com/google/refine/tests/model/ReconTypeTest.java
View File

@ -1,6 +1,8 @@
package com.google.refine.tests.model; package com.google.refine.tests.model;
import org.json.JSONException; import java.io.IOException;
import org.testng.Assert;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.model.ReconType; import com.google.refine.model.ReconType;
@ -8,9 +10,16 @@ import com.google.refine.tests.util.TestUtils;
public class ReconTypeTest { public class ReconTypeTest {
@Test @Test
public void serializeReconType() throws JSONException, Exception { public void serializeReconType() throws IOException {
String json = "{\"id\":\"Q7540126\",\"name\":\"headquarters\"}"; String json = "{\"id\":\"Q7540126\",\"name\":\"headquarters\"}";
ReconType rt = ReconType.load(json); ReconType rt = ReconType.load(json);
TestUtils.isSerializedTo(rt, json); TestUtils.isSerializedTo(rt, json);
} }
@Test
public void deserializeFromString() throws IOException {
// reconciliation services can return lists of types as bare lists of strings
ReconType rt = ReconType.load("\"Q7540126\"");
Assert.assertEquals(rt.id, "Q7540126");
}
} }