s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for the remaining commands

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-10-05 16:20:07 +01:00
parent f263d8a129
commit 9219ef36aa
5 changed files with 138 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
main/src/com/google/refine/commands/Command.java
View File

@ -46,11 +46,11 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.velocity.VelocityContext; import org.apache.velocity.VelocityContext;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet; import com.google.refine.RefineServlet;
@ -290,11 +290,11 @@ public abstract class Command {
throws IOException, JSONException { throws IOException, JSONException {
Writer w = response.getWriter(); Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w); JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w);
writer.object(); writer.writeStartObject();
writer.key("status"); writer.value(status); writer.writeStringField("status", status);
writer.key("message"); writer.value(message); writer.writeStringField("message", message);
writer.endObject(); writer.writeEndObject();
w.flush(); w.flush();
w.close(); w.close();
} }
@ -343,9 +343,14 @@ public abstract class Command {
} }
try { try {
JSONObject o = new JSONObject(); response.setCharacterEncoding("UTF-8");
o.put("code", "error"); response.setHeader("Content-Type", "application/json");
o.put("message", e.getMessage());
Writer w = response.getWriter();
JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w);
writer.writeStartObject();
writer.writeStringField("code", "error");
writer.writeStringField("message", e.getMessage());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
@ -353,11 +358,10 @@ public abstract class Command {
pw.flush(); pw.flush();
sw.flush(); sw.flush();
o.put("stack", sw.toString()); writer.writeStringField("stack", sw.toString());
writer.writeEndObject();
response.setCharacterEncoding("UTF-8"); w.flush();
response.setHeader("Content-Type", "application/json"); w.close();
respond(response, o.toString());
} catch (JSONException e1) { } catch (JSONException e1) {
e.printStackTrace(response.getWriter()); e.printStackTrace(response.getWriter());
} }

21
main/src/com/google/refine/commands/GetAllPreferencesCommand.java
View File

@ -34,14 +34,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands; package com.google.refine.commands;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore; import com.google.refine.preference.PreferenceStore;
@ -56,26 +55,16 @@ public class GetAllPreferencesCommand extends Command {
project.getMetadata().getPreferenceStore() : project.getMetadata().getPreferenceStore() :
ProjectManager.singleton.getPreferenceStore(); ProjectManager.singleton.getPreferenceStore();
try { Map<String, Object> map = new HashMap<>();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
for (String key : ps.getKeys()) { for (String key : ps.getKeys()) {
Object pref = ps.get(key); Object pref = ps.get(key);
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) { if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
writer.key(key); map.put(key, pref);
writer.value(pref);
} }
} }
writer.endObject(); respondJSON(response, map);
} catch (JSONException e) {
respondException(response, e);
}
} }
} }

94
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -55,7 +55,10 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
@ -66,6 +69,27 @@ import com.google.refine.util.ParsingUtilities;
public class GuessTypesOfColumnCommand extends Command { public class GuessTypesOfColumnCommand extends Command {
protected static class TypesResponse {
@JsonProperty("code")
protected String code;
@JsonProperty("message")
@JsonInclude(Include.NON_NULL)
protected String message;
@JsonProperty("types")
@JsonInclude(Include.NON_NULL)
List<TypeGroup> types;
protected TypesResponse(
String code,
String message,
List<TypeGroup> types) {
this.code = code;
this.message = message;
this.types = types;
}
}
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException { throws ServletException, IOException {
@ -75,35 +99,14 @@ public class GuessTypesOfColumnCommand extends Command {
String columnName = request.getParameter("columnName"); String columnName = request.getParameter("columnName");
String serviceUrl = request.getParameter("service"); String serviceUrl = request.getParameter("service");
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
Column column = project.columnModel.getColumnByName(columnName); Column column = project.columnModel.getColumnByName(columnName);
if (column == null) { if (column == null) {
writer.key("code"); writer.value("error"); respondJSON(response, new TypesResponse("error", "No such column", null));
writer.key("message"); writer.value("No such column");
} else { } else {
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl); List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
respondJSON(response, new TypesResponse("ok", null, typeGroups));
writer.key("code"); writer.value("ok");
writer.key("types"); writer.array();
for (TypeGroup tg : typeGroups) {
writer.object();
writer.key("id"); writer.value(tg.id);
writer.key("name"); writer.value(tg.name);
writer.key("score"); writer.value(tg.score);
writer.key("count"); writer.value(tg.count);
writer.endObject();
} }
writer.endArray();
}
writer.endObject();
} catch (Exception e) { } catch (Exception e) {
respondException(response, e); respondException(response, e);
} }
@ -111,6 +114,18 @@ public class GuessTypesOfColumnCommand extends Command {
final static int SAMPLE_SIZE = 10; final static int SAMPLE_SIZE = 10;
protected static class IndividualQuery {
@JsonProperty("query")
protected String query;
@JsonProperty("limit")
protected int limit;
protected IndividualQuery(String query, int limit) {
this.query = query;
this.limit = limit;
}
}
/** /**
* Run relevance searches for the first n cells in the given column and * Run relevance searches for the first n cells in the given column and
* count the types of the results. Return a sorted list of types, from most * count the types of the results. Return a sorted list of types, from most
@ -144,25 +159,12 @@ public class GuessTypesOfColumnCommand extends Command {
} }
} }
StringWriter stringWriter = new StringWriter(); Map<String, IndividualQuery> queryMap = new HashMap<>();
try {
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
for (int i = 0; i < samples.size(); i++) { for (int i = 0; i < samples.size(); i++) {
jsonWriter.key("q" + i); queryMap.put("q" + i, new IndividualQuery(samples.get(i), 3));
jsonWriter.object();
jsonWriter.key("query"); jsonWriter.value(samples.get(i));
jsonWriter.key("limit"); jsonWriter.value(3);
jsonWriter.endObject();
}
jsonWriter.endObject();
} catch (JSONException e) {
logger.error("Error constructing query", e);
} }
String queriesString = stringWriter.toString(); String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap);
try { try {
URL url = new URL(serviceUrl); URL url = new URL(serviceUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@ -264,10 +266,14 @@ public class GuessTypesOfColumnCommand extends Command {
} }
static protected class TypeGroup { static protected class TypeGroup {
String id; @JsonProperty("id")
String name; protected String id;
int count; @JsonProperty("name")
double score; protected String name;
@JsonProperty("count")
protected int count;
@JsonProperty("score")
protected double score;
TypeGroup(String id, String name, double score) { TypeGroup(String id, String name, double score) {
this.id = id; this.id = id;

70
main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java
View File

@ -46,7 +46,8 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
@ -64,6 +65,19 @@ import com.google.refine.util.ParsingUtilities;
public class PreviewExtendDataCommand extends Command { public class PreviewExtendDataCommand extends Command {
protected static class PreviewResponse {
public PreviewResponse(List<ColumnInfo> columns2, List<List<Object>> rows2) {
columns = columns2;
rows = rows2;
}
@JsonProperty("code")
protected String code = "ok";
@JsonProperty("columns")
protected List<ColumnInfo> columns;
@JsonProperty("rows")
protected List<List<Object>> rows;
}
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException { throws ServletException, IOException {
@ -122,27 +136,8 @@ public class PreviewExtendDataCommand extends Command {
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>(); Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint); ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap); Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
List<List<Object>> rows = new ArrayList<>();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("code"); writer.value("ok");
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.key("name");
writer.value(info.name);
writer.key("id");
writer.value(info.id);
writer.endObject();
}
writer.endArray();
writer.key("rows");
writer.array();
for (int r = 0; r < topicNames.size(); r++) { for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r); String id = topicIds.get(r);
String topicName = topicNames.get(r); String topicName = topicNames.get(r);
@ -153,46 +148,33 @@ public class PreviewExtendDataCommand extends Command {
if (ext.data.length > 0) { if (ext.data.length > 0) {
for (Object[] row : ext.data) { for (Object[] row : ext.data) {
writer.array(); List<Object> jsonRow = new ArrayList<>();
if (first) { if (first) {
writer.value(topicName); jsonRow.add(topicName);
first = false; first = false;
} else { } else {
writer.value(null); jsonRow.add(null);
} }
for (Object cell : row) { for (Object cell : row) {
if (cell != null && cell instanceof ReconCandidate) { jsonRow.add(cell);
ReconCandidate rc = (ReconCandidate) cell;
writer.object();
writer.key("id"); writer.value(rc.id);
writer.key("name"); writer.value(rc.name);
writer.endObject();
} else {
writer.value(cell);
} }
} rows.add(jsonRow);
writer.endArray();
} }
continue; continue;
} }
} }
writer.array(); List<Object> supplement = new ArrayList<>();
if (id != null) { if (id != null) {
writer.object(); supplement.add(new ReconCandidate(id, topicName, new String[0], 100));
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(topicName);
writer.endObject();
} else { } else {
writer.value("<not reconciled>"); supplement.add("<not reconciled>");
} }
writer.endArray(); rows.add(supplement);
} }
writer.endArray();
writer.endObject(); respondJSON(response, new PreviewResponse(job.columns, rows));
} catch (Exception e) { } catch (Exception e) {
respondException(response, e); respondException(response, e);
} }

5
main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java
View File

@ -141,8 +141,11 @@ public class ReconciledDataExtensionJob {
} }
} }
// Json serialization is used in PreviewExtendDataCommand
static public class ColumnInfo { static public class ColumnInfo {
@JsonProperty("name")
final public String name; final public String name;
@JsonProperty("id")
final public String id; final public String id;
final public ReconType expectedType; final public ReconType expectedType;
@ -320,7 +323,7 @@ public class ReconciledDataExtensionJob {
static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws IOException { static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws IOException {
DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties); DataExtensionQuery query = new DataExtensionQuery(ids.stream().filter(e -> e != null).collect(Collectors.toList()), node.properties);
ParsingUtilities.saveWriter.writeValue(writer, query); ParsingUtilities.saveWriter.writeValue(writer, query);
} }