diff --git a/main/src/com/google/refine/commands/Command.java b/main/src/com/google/refine/commands/Command.java index 39415f4d9..7664c977c 100644 --- a/main/src/com/google/refine/commands/Command.java +++ b/main/src/com/google/refine/commands/Command.java @@ -46,11 +46,11 @@ import javax.servlet.http.HttpServletResponse; import org.apache.velocity.VelocityContext; import org.json.JSONException; import org.json.JSONObject; -import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; @@ -290,11 +290,11 @@ public abstract class Command { throws IOException, JSONException { Writer w = response.getWriter(); - JSONWriter writer = new JSONWriter(w); - writer.object(); - writer.key("status"); writer.value(status); - writer.key("message"); writer.value(message); - writer.endObject(); + JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); + writer.writeStartObject(); + writer.writeStringField("status", status); + writer.writeStringField("message", message); + writer.writeEndObject(); w.flush(); w.close(); } @@ -343,9 +343,14 @@ public abstract class Command { } try { - JSONObject o = new JSONObject(); - o.put("code", "error"); - o.put("message", e.getMessage()); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Writer w = response.getWriter(); + JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); + writer.writeStartObject(); + writer.writeStringField("code", "error"); + writer.writeStringField("message", e.getMessage()); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -353,11 +358,10 @@ public abstract class Command { pw.flush(); sw.flush(); - o.put("stack", sw.toString()); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - respond(response, o.toString()); + writer.writeStringField("stack", sw.toString()); + writer.writeEndObject(); + w.flush(); + w.close(); } catch (JSONException e1) { e.printStackTrace(response.getWriter()); } diff --git a/main/src/com/google/refine/commands/GetAllPreferencesCommand.java b/main/src/com/google/refine/commands/GetAllPreferencesCommand.java index 8b2c6bc74..cdf641e08 100644 --- a/main/src/com/google/refine/commands/GetAllPreferencesCommand.java +++ b/main/src/com/google/refine/commands/GetAllPreferencesCommand.java @@ -34,14 +34,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.commands; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONException; -import org.json.JSONWriter; - import com.google.refine.ProjectManager; import com.google.refine.model.Project; import com.google.refine.preference.PreferenceStore; @@ -56,26 +55,16 @@ public class GetAllPreferencesCommand extends Command { project.getMetadata().getPreferenceStore() : ProjectManager.singleton.getPreferenceStore(); - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - - writer.object(); - - for (String key : ps.getKeys()) { - Object pref = ps.get(key); - if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) { - writer.key(key); - writer.value(pref); - } + Map map = new HashMap<>(); + + for (String key : ps.getKeys()) { + Object pref = ps.get(key); + if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) { + map.put(key, pref); } - - writer.endObject(); - } catch (JSONException e) { - respondException(response, e); } + + respondJSON(response, map); } } diff --git a/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java b/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java index 3d920d145..9759a80ae 100644 --- a/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java +++ b/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java @@ -55,7 +55,10 @@ import javax.servlet.http.HttpServletResponse; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.json.JSONWriter; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.refine.commands.Command; import com.google.refine.expr.ExpressionUtils; @@ -66,6 +69,27 @@ import com.google.refine.util.ParsingUtilities; public class GuessTypesOfColumnCommand extends Command { + + protected static class TypesResponse { + @JsonProperty("code") + protected String code; + @JsonProperty("message") + @JsonInclude(Include.NON_NULL) + protected String message; + @JsonProperty("types") + @JsonInclude(Include.NON_NULL) + List types; + + protected TypesResponse( + String code, + String message, + List types) { + this.code = code; + this.message = message; + this.types = types; + } + } + @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -75,35 +99,14 @@ public class GuessTypesOfColumnCommand extends Command { String columnName = request.getParameter("columnName"); String serviceUrl = request.getParameter("service"); - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - Column column = project.columnModel.getColumnByName(columnName); if (column == null) { - writer.key("code"); writer.value("error"); - writer.key("message"); writer.value("No such column"); + respondJSON(response, new TypesResponse("error", "No such column", null)); } else { - List typeGroups = guessTypes(project, column, serviceUrl); - - writer.key("code"); writer.value("ok"); - writer.key("types"); writer.array(); - - for (TypeGroup tg : typeGroups) { - writer.object(); - writer.key("id"); writer.value(tg.id); - writer.key("name"); writer.value(tg.name); - writer.key("score"); writer.value(tg.score); - writer.key("count"); writer.value(tg.count); - writer.endObject(); - } - - writer.endArray(); + List typeGroups = guessTypes(project, column, serviceUrl); + respondJSON(response, new TypesResponse("ok", null, typeGroups)); } - - writer.endObject(); + } catch (Exception e) { respondException(response, e); } @@ -111,6 +114,18 @@ public class GuessTypesOfColumnCommand extends Command { final static int SAMPLE_SIZE = 10; + protected static class IndividualQuery { + @JsonProperty("query") + protected String query; + @JsonProperty("limit") + protected int limit; + + protected IndividualQuery(String query, int limit) { + this.query = query; + this.limit = limit; + } + } + /** * Run relevance searches for the first n cells in the given column and * count the types of the results. Return a sorted list of types, from most @@ -144,25 +159,12 @@ public class GuessTypesOfColumnCommand extends Command { } } - StringWriter stringWriter = new StringWriter(); - try { - JSONWriter jsonWriter = new JSONWriter(stringWriter); - jsonWriter.object(); - for (int i = 0; i < samples.size(); i++) { - jsonWriter.key("q" + i); - jsonWriter.object(); - - jsonWriter.key("query"); jsonWriter.value(samples.get(i)); - jsonWriter.key("limit"); jsonWriter.value(3); - - jsonWriter.endObject(); - } - jsonWriter.endObject(); - } catch (JSONException e) { - logger.error("Error constructing query", e); + Map queryMap = new HashMap<>(); + for (int i = 0; i < samples.size(); i++) { + queryMap.put("q" + i, new IndividualQuery(samples.get(i), 3)); } - String queriesString = stringWriter.toString(); + String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap); try { URL url = new URL(serviceUrl); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); @@ -264,10 +266,14 @@ public class GuessTypesOfColumnCommand extends Command { } static protected class TypeGroup { - String id; - String name; - int count; - double score; + @JsonProperty("id") + protected String id; + @JsonProperty("name") + protected String name; + @JsonProperty("count") + protected int count; + @JsonProperty("score") + protected double score; TypeGroup(String id, String name, double score) { this.id = id; diff --git a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java index f0c6906a7..c8d13a2aa 100644 --- a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java +++ b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java @@ -46,7 +46,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.json.JSONArray; -import org.json.JSONWriter; + +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.refine.commands.Command; import com.google.refine.model.Cell; @@ -64,6 +65,19 @@ import com.google.refine.util.ParsingUtilities; public class PreviewExtendDataCommand extends Command { + protected static class PreviewResponse { + public PreviewResponse(List columns2, List> rows2) { + columns = columns2; + rows = rows2; + } + @JsonProperty("code") + protected String code = "ok"; + @JsonProperty("columns") + protected List columns; + @JsonProperty("rows") + protected List> rows; + } + @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -122,77 +136,45 @@ public class PreviewExtendDataCommand extends Command { Map reconCandidateMap = new HashMap(); ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint); Map map = job.extend(ids, reconCandidateMap); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - writer.key("code"); writer.value("ok"); - writer.key("columns"); - writer.array(); - for (ColumnInfo info : job.columns) { - writer.object(); - writer.key("name"); - writer.value(info.name); - writer.key("id"); - writer.value(info.id); - writer.endObject(); - } - writer.endArray(); - - writer.key("rows"); - writer.array(); - for (int r = 0; r < topicNames.size(); r++) { - String id = topicIds.get(r); - String topicName = topicNames.get(r); - - if (id != null && map.containsKey(id)) { - DataExtension ext = map.get(id); - boolean first = true; - - if (ext.data.length > 0) { - for (Object[] row : ext.data) { - writer.array(); - if (first) { - writer.value(topicName); - first = false; - } else { - writer.value(null); - } - - for (Object cell : row) { - if (cell != null && cell instanceof ReconCandidate) { - ReconCandidate rc = (ReconCandidate) cell; - writer.object(); - writer.key("id"); writer.value(rc.id); - writer.key("name"); writer.value(rc.name); - writer.endObject(); - } else { - writer.value(cell); - } - } - - writer.endArray(); - } - continue; - } - } - - writer.array(); - if (id != null) { - writer.object(); - writer.key("id"); writer.value(id); - writer.key("name"); writer.value(topicName); - writer.endObject(); - } else { - writer.value(""); - } - writer.endArray(); - } - writer.endArray(); + List> rows = new ArrayList<>(); + + for (int r = 0; r < topicNames.size(); r++) { + String id = topicIds.get(r); + String topicName = topicNames.get(r); - writer.endObject(); + if (id != null && map.containsKey(id)) { + DataExtension ext = map.get(id); + boolean first = true; + + if (ext.data.length > 0) { + for (Object[] row : ext.data) { + List jsonRow = new ArrayList<>(); + if (first) { + jsonRow.add(topicName); + first = false; + } else { + jsonRow.add(null); + } + + for (Object cell : row) { + jsonRow.add(cell); + } + rows.add(jsonRow); + } + continue; + } + } + + List supplement = new ArrayList<>(); + if (id != null) { + supplement.add(new ReconCandidate(id, topicName, new String[0], 100)); + } else { + supplement.add(""); + } + rows.add(supplement); + } + + respondJSON(response, new PreviewResponse(job.columns, rows)); } catch (Exception e) { respondException(response, e); } diff --git a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java index 83c78e538..47f575600 100644 --- a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java +++ b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java @@ -141,8 +141,11 @@ public class ReconciledDataExtensionJob { } } + // Json serialization is used in PreviewExtendDataCommand static public class ColumnInfo { + @JsonProperty("name") final public String name; + @JsonProperty("id") final public String id; final public ReconType expectedType; @@ -320,7 +323,7 @@ public class ReconciledDataExtensionJob { static protected void formulateQuery(Set ids, DataExtensionConfig node, Writer writer) throws IOException { - DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties); + DataExtensionQuery query = new DataExtensionQuery(ids.stream().filter(e -> e != null).collect(Collectors.toList()), node.properties); ParsingUtilities.saveWriter.writeValue(writer, query); }