s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for the remaining commands

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-10-05 16:20:07 +01:00
parent f263d8a129
commit 9219ef36aa
5 changed files with 138 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
main/src/com/google/refine/commands/Command.java
View File

@ -46,11 +46,11 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.velocity.VelocityContext;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
@ -290,11 +290,11 @@ public abstract class Command {
throws IOException, JSONException {
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
writer.object();
writer.key("status"); writer.value(status);
writer.key("message"); writer.value(message);
writer.endObject();
JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w);
writer.writeStartObject();
writer.writeStringField("status", status);
writer.writeStringField("message", message);
writer.writeEndObject();
w.flush();
w.close();
}
@ -343,9 +343,14 @@ public abstract class Command {
}
try {
JSONObject o = new JSONObject();
o.put("code", "error");
o.put("message", e.getMessage());
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
Writer w = response.getWriter();
JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w);
writer.writeStartObject();
writer.writeStringField("code", "error");
writer.writeStringField("message", e.getMessage());
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
@ -353,11 +358,10 @@ public abstract class Command {
pw.flush();
sw.flush();
o.put("stack", sw.toString());
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
respond(response, o.toString());
writer.writeStringField("stack", sw.toString());
writer.writeEndObject();
w.flush();
w.close();
} catch (JSONException e1) {
e.printStackTrace(response.getWriter());
}

31
main/src/com/google/refine/commands/GetAllPreferencesCommand.java
View File

@ -34,14 +34,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.ProjectManager;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
@ -56,26 +55,16 @@ public class GetAllPreferencesCommand extends Command {
project.getMetadata().getPreferenceStore() :
ProjectManager.singleton.getPreferenceStore();
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
for (String key : ps.getKeys()) {
Object pref = ps.get(key);
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
writer.key(key);
writer.value(pref);
}
Map<String, Object> map = new HashMap<>();
for (String key : ps.getKeys()) {
Object pref = ps.get(key);
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
map.put(key, pref);
}
writer.endObject();
} catch (JSONException e) {
respondException(response, e);
}
respondJSON(response, map);
}
}

100
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -55,7 +55,10 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils;
@ -66,6 +69,27 @@ import com.google.refine.util.ParsingUtilities;
public class GuessTypesOfColumnCommand extends Command {
protected static class TypesResponse {
@JsonProperty("code")
protected String code;
@JsonProperty("message")
@JsonInclude(Include.NON_NULL)
protected String message;
@JsonProperty("types")
@JsonInclude(Include.NON_NULL)
List<TypeGroup> types;
protected TypesResponse(
String code,
String message,
List<TypeGroup> types) {
this.code = code;
this.message = message;
this.types = types;
}
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -75,35 +99,14 @@ public class GuessTypesOfColumnCommand extends Command {
String columnName = request.getParameter("columnName");
String serviceUrl = request.getParameter("service");
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
Column column = project.columnModel.getColumnByName(columnName);
if (column == null) {
writer.key("code"); writer.value("error");
writer.key("message"); writer.value("No such column");
respondJSON(response, new TypesResponse("error", "No such column", null));
} else {
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
writer.key("code"); writer.value("ok");
writer.key("types"); writer.array();
for (TypeGroup tg : typeGroups) {
writer.object();
writer.key("id"); writer.value(tg.id);
writer.key("name"); writer.value(tg.name);
writer.key("score"); writer.value(tg.score);
writer.key("count"); writer.value(tg.count);
writer.endObject();
}
writer.endArray();
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
respondJSON(response, new TypesResponse("ok", null, typeGroups));
}
writer.endObject();
} catch (Exception e) {
respondException(response, e);
}
@ -111,6 +114,18 @@ public class GuessTypesOfColumnCommand extends Command {
final static int SAMPLE_SIZE = 10;
protected static class IndividualQuery {
@JsonProperty("query")
protected String query;
@JsonProperty("limit")
protected int limit;
protected IndividualQuery(String query, int limit) {
this.query = query;
this.limit = limit;
}
}
/**
* Run relevance searches for the first n cells in the given column and
* count the types of the results. Return a sorted list of types, from most
@ -144,25 +159,12 @@ public class GuessTypesOfColumnCommand extends Command {
}
}
StringWriter stringWriter = new StringWriter();
try {
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
for (int i = 0; i < samples.size(); i++) {
jsonWriter.key("q" + i);
jsonWriter.object();
jsonWriter.key("query"); jsonWriter.value(samples.get(i));
jsonWriter.key("limit"); jsonWriter.value(3);
jsonWriter.endObject();
}
jsonWriter.endObject();
} catch (JSONException e) {
logger.error("Error constructing query", e);
Map<String, IndividualQuery> queryMap = new HashMap<>();
for (int i = 0; i < samples.size(); i++) {
queryMap.put("q" + i, new IndividualQuery(samples.get(i), 3));
}
String queriesString = stringWriter.toString();
String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap);
try {
URL url = new URL(serviceUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@ -264,10 +266,14 @@ public class GuessTypesOfColumnCommand extends Command {
}
static protected class TypeGroup {
String id;
String name;
int count;
double score;
@JsonProperty("id")
protected String id;
@JsonProperty("name")
protected String name;
@JsonProperty("count")
protected int count;
@JsonProperty("score")
protected double score;
TypeGroup(String id, String name, double score) {
this.id = id;

124
main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java
View File

@ -46,7 +46,8 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.model.Cell;
@ -64,6 +65,19 @@ import com.google.refine.util.ParsingUtilities;
public class PreviewExtendDataCommand extends Command {
protected static class PreviewResponse {
public PreviewResponse(List<ColumnInfo> columns2, List<List<Object>> rows2) {
columns = columns2;
rows = rows2;
}
@JsonProperty("code")
protected String code = "ok";
@JsonProperty("columns")
protected List<ColumnInfo> columns;
@JsonProperty("rows")
protected List<List<Object>> rows;
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -122,77 +136,45 @@ public class PreviewExtendDataCommand extends Command {
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("code"); writer.value("ok");
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.key("name");
writer.value(info.name);
writer.key("id");
writer.value(info.id);
writer.endObject();
}
writer.endArray();
writer.key("rows");
writer.array();
for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r);
String topicName = topicNames.get(r);
if (id != null && map.containsKey(id)) {
DataExtension ext = map.get(id);
boolean first = true;
if (ext.data.length > 0) {
for (Object[] row : ext.data) {
writer.array();
if (first) {
writer.value(topicName);
first = false;
} else {
writer.value(null);
}
for (Object cell : row) {
if (cell != null && cell instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) cell;
writer.object();
writer.key("id"); writer.value(rc.id);
writer.key("name"); writer.value(rc.name);
writer.endObject();
} else {
writer.value(cell);
}
}
writer.endArray();
}
continue;
}
}
writer.array();
if (id != null) {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(topicName);
writer.endObject();
} else {
writer.value("<not reconciled>");
}
writer.endArray();
}
writer.endArray();
List<List<Object>> rows = new ArrayList<>();
for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r);
String topicName = topicNames.get(r);
writer.endObject();
if (id != null && map.containsKey(id)) {
DataExtension ext = map.get(id);
boolean first = true;
if (ext.data.length > 0) {
for (Object[] row : ext.data) {
List<Object> jsonRow = new ArrayList<>();
if (first) {
jsonRow.add(topicName);
first = false;
} else {
jsonRow.add(null);
}
for (Object cell : row) {
jsonRow.add(cell);
}
rows.add(jsonRow);
}
continue;
}
}
List<Object> supplement = new ArrayList<>();
if (id != null) {
supplement.add(new ReconCandidate(id, topicName, new String[0], 100));
} else {
supplement.add("<not reconciled>");
}
rows.add(supplement);
}
respondJSON(response, new PreviewResponse(job.columns, rows));
} catch (Exception e) {
respondException(response, e);
}

5
main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java
View File

@ -141,8 +141,11 @@ public class ReconciledDataExtensionJob {
}
}
// Json serialization is used in PreviewExtendDataCommand
static public class ColumnInfo {
@JsonProperty("name")
final public String name;
@JsonProperty("id")
final public String id;
final public ReconType expectedType;
@ -320,7 +323,7 @@ public class ReconciledDataExtensionJob {
static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws IOException {
DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties);
DataExtensionQuery query = new DataExtensionQuery(ids.stream().filter(e -> e != null).collect(Collectors.toList()), node.properties);
ParsingUtilities.saveWriter.writeValue(writer, query);
}