diff --git a/src/main/java/com/metaweb/gridworks/GridworksServlet.java b/src/main/java/com/metaweb/gridworks/GridworksServlet.java index d51428f52..2c6554012 100644 --- a/src/main/java/com/metaweb/gridworks/GridworksServlet.java +++ b/src/main/java/com/metaweb/gridworks/GridworksServlet.java @@ -51,6 +51,7 @@ import com.metaweb.gridworks.commands.util.GetExpressionLanguageInfoCommand; import com.metaweb.gridworks.commands.util.GuessTypesOfColumnCommand; import com.metaweb.gridworks.commands.util.LogExpressionCommand; import com.metaweb.gridworks.commands.util.PreviewExpressionCommand; +import com.metaweb.gridworks.commands.util.PreviewExtendDataCommand; import com.metaweb.gridworks.commands.util.PreviewProtographCommand; public class GridworksServlet extends HttpServlet { @@ -106,12 +107,14 @@ public class GridworksServlet extends HttpServlet { _commands.put("save-protograph", new SaveProtographCommand()); - _commands.put("preview-expression", new PreviewExpressionCommand()); _commands.put("get-expression-language-info", new GetExpressionLanguageInfoCommand()); _commands.put("get-expression-history", new GetExpressionHistoryCommand()); _commands.put("log-expression", new LogExpressionCommand()); + _commands.put("preview-expression", new PreviewExpressionCommand()); + _commands.put("preview-extend-data", new PreviewExtendDataCommand()); _commands.put("preview-protograph", new PreviewProtographCommand()); + _commands.put("guess-types-of-column", new GuessTypesOfColumnCommand()); } diff --git a/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java new file mode 100644 index 000000000..32d5276cb --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java @@ -0,0 +1,129 @@ +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; + +public class PreviewExtendDataCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + String columnName = request.getParameter("columnName"); + + String rowIndicesString = request.getParameter("rowIndices"); + if (rowIndicesString == null) { + respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); + return; + } + + String jsonString = request.getParameter("extension"); + JSONObject json = jsonStringToObject(jsonString); + + JSONArray rowIndices = jsonStringToArray(rowIndicesString); + int length = rowIndices.length(); + int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex(); + + List topicNames = new ArrayList(); + List topicGuids = new ArrayList(); + Set guids = new HashSet(); + for (int i = 0; i < length; i++) { + int rowIndex = rowIndices.getInt(i); + if (rowIndex >= 0 && rowIndex < project.rows.size()) { + Row row = project.rows.get(rowIndex); + Cell cell = row.getCell(cellIndex); + if (cell != null && cell.recon != null && cell.recon.match != null) { + topicNames.add(cell.recon.match.topicName); + topicGuids.add(cell.recon.match.topicGUID); + guids.add(cell.recon.match.topicGUID); + } + } + } + + FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json); + Map map = job.extend(guids); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + writer.key("columns"); + writer.array(); + for (String name : job.columnNames) { + writer.value(name); + } + writer.endArray(); + + writer.key("rows"); + writer.array(); + for (int r = 0; r < topicNames.size(); r++) { + String guid = topicGuids.get(r); + String topicName = topicNames.get(r); + + if (map.containsKey(guid)) { + DataExtension ext = map.get(guid); + boolean first = true; + + if (ext.data.length > 0) { + for (Object[] row : ext.data) { + writer.array(); + if (first) { + writer.value(topicName); + first = false; + } else { + writer.value(null); + } + + for (Object cell : row) { + if (cell != null && cell instanceof ReconCandidate) { + ReconCandidate rc = (ReconCandidate) cell; + writer.object(); + writer.key("id"); writer.value(rc.topicID); + writer.key("name"); writer.value(rc.topicName); + writer.endObject(); + } else { + writer.value(cell); + } + } + + writer.endArray(); + } + continue; + } + } + + writer.array(); + writer.value(topicName); + writer.endArray(); + } + writer.endArray(); + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java new file mode 100644 index 000000000..83bdb803a --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java @@ -0,0 +1,310 @@ +/** + * + */ +package com.metaweb.gridworks.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.io.StringWriter; +import java.io.Writer; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.ReconCandidate; + +public class FreebaseDataExtensionJob { + static public class DataExtension { + final public Object[][] data; + + protected DataExtension(Object[][] data) { + this.data = data; + } + } + + final public JSONObject extension; + final public int columnCount; + final public List columnNames = new ArrayList(); + + public FreebaseDataExtensionJob(JSONObject obj) throws JSONException { + this.extension = obj; + this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ? + countColumns(obj.getJSONArray("properties"), columnNames) : 0; + } + + public Map extend(Set guids) throws Exception { + StringWriter writer = new StringWriter(); + formulateQuery(guids, extension, writer); + + URL url = new URL("http://api.freebase.com/api/service/mqlread"); + InputStream is = doPost(url, "query", writer.toString()); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + + Map map = new HashMap(); + JSONArray a = o.getJSONArray("result"); + int l = a.length(); + + for (int i = 0; i < l; i++) { + JSONObject o2 = a.getJSONObject(i); + String guid = o2.getString("guid"); + FreebaseDataExtensionJob.DataExtension ext = collectResult(o2); + + if (ext != null) { + map.put(guid, ext); + } + } + + return map; + } finally { + is.close(); + } + } + + protected FreebaseDataExtensionJob.DataExtension collectResult(JSONObject obj) throws JSONException { + List rows = new ArrayList(); + + collectResult(rows, extension, obj, 0, 0); + + Object[][] data = new Object[rows.size()][columnCount]; + rows.toArray(data); + + return new DataExtension(data); + } + + protected void storeCell( + List rows, + int row, + int col, + Object value + ) { + while (row >= rows.size()) { + rows.add(new Object[columnCount]); + } + rows.get(row)[col] = value; + } + + protected void storeCell( + List rows, + int row, + int col, + JSONObject obj + ) throws JSONException { + storeCell(rows, row, col, + new ReconCandidate( + obj.getString("id"), + obj.getString("guid"), + obj.getString("name"), + JSONUtilities.getStringArray(obj, "type"), + 100 + ) + ); + } + + protected int[] collectResult( + List rows, + JSONObject extNode, + JSONObject resultNode, + int startRowIndex, + int startColumnIndex + ) throws JSONException { + String propertyID = extNode.getString("id"); + String expectedTypeID = extNode.getString("expected"); + + JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ? + resultNode.getJSONArray(propertyID) : null; + + if (expectedTypeID.startsWith("/type/")) { + if (a != null) { + int l = a.length(); + for (int r = 0; r < l; r++) { + Object o = a.isNull(r) ? null : a.get(r); + if (o instanceof Serializable) { + storeCell(rows, startRowIndex++, startColumnIndex, o); + } + } + } + + // note that we still take up a column even if we don't have any data + return new int[] { startRowIndex, startColumnIndex + 1 }; + } else { + boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties")); + boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included")); + + if (a != null) { + int maxColIndex = startColumnIndex; + + int l = a.length(); + for (int r = 0; r < l; r++) { + JSONObject o = a.isNull(r) ? null : a.getJSONObject(r); + + int startColumnIndex2 = startColumnIndex; + int startRowIndex2 = startRowIndex; + + if (isOwnColumn) { + storeCell(rows, startRowIndex2++, startColumnIndex2++, o); + } + + int[] rowcol = collectResult( + rows, + extNode.getJSONArray("properties"), + o, + startRowIndex, + startColumnIndex2 + ); + + startRowIndex = rowcol[0]; + maxColIndex = Math.max(maxColIndex, rowcol[1]); + } + + return new int[] { startRowIndex, maxColIndex }; + } else { + return new int[] { + startRowIndex, + countColumns(extNode, null) + }; + } + } + } + + protected int[] collectResult( + List rows, + JSONArray subProperties, + JSONObject resultNode, + int startRowIndex, + int startColumnIndex + ) throws JSONException { + int maxStartRowIndex = startRowIndex; + + int k = subProperties.length(); + for (int c = 0; c < k; c++) { + int[] rowcol = collectResult( + rows, + subProperties.getJSONObject(c), + resultNode, + startRowIndex, + startColumnIndex + ); + + maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]); + startColumnIndex = rowcol[1]; + } + + return new int[] { maxStartRowIndex, startColumnIndex }; + } + + +static protected InputStream doPost(URL url, String name, String load) throws IOException { + URLConnection connection = url.openConnection(); + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + connection.setConnectTimeout(5000); + + DataOutputStream dos = new DataOutputStream(connection.getOutputStream()); + try { + String data = name + "=" + ParsingUtilities.encode(load); + + dos.writeBytes(data); + } finally { + dos.flush(); + dos.close(); + } + + connection.connect(); + + return connection.getInputStream(); +} + +static protected void formulateQuery(Set guids, JSONObject node, Writer writer) throws JSONException { + JSONWriter jsonWriter = new JSONWriter(writer); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("guid|="); + jsonWriter.array(); + for (String guid : guids) { + jsonWriter.value(guid); + } + jsonWriter.endArray(); + + formulateQueryNode(node, jsonWriter); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); +} + +static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException { + String propertyID = node.getString("id"); + String expectedTypeID = node.getString("expected"); + + writer.key(propertyID); + writer.array(); + { + if (!expectedTypeID.startsWith("/type/")) { // not literal + writer.object(); + writer.key("limit"); writer.value(10); + { + boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); + + if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) { + writer.key("name"); writer.value(null); + writer.key("id"); writer.value(null); + writer.key("guid"); writer.value(null); + writer.key("type"); writer.array(); writer.endArray(); + } + + if (hasSubProperties) { + JSONArray a = node.getJSONArray("properties"); + int l = a.length(); + + for (int i = 0; i < l; i++) { + formulateQueryNode(a.getJSONObject(i), writer); + } + } + } + writer.endObject(); + } + } + writer.endArray(); +} + +static protected int countColumns(JSONObject obj, List columnNames) throws JSONException { + if (obj.has("properties") && !obj.isNull("properties")) { + boolean included = (obj.has("included") && obj.getBoolean("included")); + if (included && columnNames != null) { + columnNames.add(obj.getString("name")); + } + return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames); + } else { + if (columnNames != null) { + columnNames.add(obj.getString("name")); + } + return 1; + } +} + +static protected int countColumns(JSONArray a, List columnNames) throws JSONException { + int c = 0; + int l = a.length(); + for (int i = 0; i < l; i++) { + c += countColumns(a.getJSONObject(i), columnNames); + } + return c; +} +} \ No newline at end of file diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index 365dec9ea..814d83635 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -1 +1 @@ - Gridworks
Gridworks
Gridworks

Upload Data File

Data File:
Project Name:
Load up to: data rows (optional)
Skip: initial data rows (optional)

Import Existing Project

Project TAR File:
Re-name Project: (optional)
\ No newline at end of file + Gridworks
Gridworks
Gridworks

Upload Data File

Data File:
Project Name:
Load up to: data rows (optional)
Skip: initial data rows (optional)

Import Existing Project

Project TAR File:
Re-name Project: (optional)
\ No newline at end of file diff --git a/src/main/webapp/project.html b/src/main/webapp/project.html index 384f62026..122f1a955 100644 --- a/src/main/webapp/project.html +++ b/src/main/webapp/project.html @@ -1 +1 @@ - Gridworks
starting up ...
\ No newline at end of file + Gridworks
starting up ...
\ No newline at end of file diff --git a/src/main/webapp/scripts/dialogs/expression-preview-dialog.js b/src/main/webapp/scripts/dialogs/expression-preview-dialog.js index bc868fc45..dad8830eb 100644 --- a/src/main/webapp/scripts/dialogs/expression-preview-dialog.js +++ b/src/main/webapp/scripts/dialogs/expression-preview-dialog.js @@ -32,13 +32,13 @@ function ExpressionPreviewDialog(title, cellIndex, rowIndices, values, expressio }; ExpressionPreviewDialog.generateWidgetHtml = function() { - return '' + + return '
' + '' + '' + '' + '' + '' + - '
ExpressionLanguage