More work on the extend data preview dialog. Results now looks correct, but we still are not handling CVTs.
git-svn-id: http://google-refine.googlecode.com/svn/trunk@295 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh
parent
e35c4c3b94
commit
a32273de70
@ -47,8 +47,7 @@ public class FreebaseDataExtensionJob {
|
||||
StringWriter writer = new StringWriter();
|
||||
formulateQuery(guids, extension, writer);
|
||||
|
||||
URL url = new URL("http://api.freebase.com/api/service/mqlread");
|
||||
InputStream is = doPost(url, "query", writer.toString());
|
||||
InputStream is = doMqlRead(writer.toString());
|
||||
try {
|
||||
String s = ParsingUtilities.inputStreamToString(is);
|
||||
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
||||
@ -143,7 +142,7 @@ public class FreebaseDataExtensionJob {
|
||||
boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
|
||||
boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
|
||||
|
||||
if (a != null) {
|
||||
if (a != null && a.length() > 0) {
|
||||
int maxColIndex = startColumnIndex;
|
||||
|
||||
int l = a.length();
|
||||
@ -166,16 +165,19 @@ public class FreebaseDataExtensionJob {
|
||||
startColumnIndex2
|
||||
);
|
||||
|
||||
startRowIndex = rowcol[0];
|
||||
maxColIndex = Math.max(maxColIndex, rowcol[1]);
|
||||
startRowIndex2 = rowcol[0];
|
||||
startColumnIndex2 = rowcol[1];
|
||||
}
|
||||
|
||||
startRowIndex = startRowIndex2;
|
||||
maxColIndex = Math.max(maxColIndex, startColumnIndex2);
|
||||
}
|
||||
|
||||
return new int[] { startRowIndex, maxColIndex };
|
||||
} else {
|
||||
return new int[] {
|
||||
startRowIndex,
|
||||
countColumns(extNode, null)
|
||||
startColumnIndex + countColumns(extNode, null)
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -208,109 +210,112 @@ public class FreebaseDataExtensionJob {
|
||||
}
|
||||
|
||||
|
||||
static protected InputStream doPost(URL url, String name, String load) throws IOException {
|
||||
URLConnection connection = url.openConnection();
|
||||
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
||||
connection.setConnectTimeout(5000);
|
||||
connection.setDoOutput(true);
|
||||
|
||||
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
|
||||
try {
|
||||
String data = name + "=" + ParsingUtilities.encode(load);
|
||||
static protected InputStream doMqlRead(String query) throws IOException {
|
||||
URL url = new URL("http://api.freebase.com/api/service/mqlread");
|
||||
|
||||
URLConnection connection = url.openConnection();
|
||||
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
||||
connection.setConnectTimeout(5000);
|
||||
connection.setDoOutput(true);
|
||||
|
||||
dos.writeBytes(data);
|
||||
} finally {
|
||||
dos.flush();
|
||||
dos.close();
|
||||
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
|
||||
try {
|
||||
String body = "extended=1&query=" + ParsingUtilities.encode(query);
|
||||
|
||||
dos.writeBytes(body);
|
||||
} finally {
|
||||
dos.flush();
|
||||
dos.close();
|
||||
}
|
||||
|
||||
connection.connect();
|
||||
|
||||
return connection.getInputStream();
|
||||
}
|
||||
|
||||
connection.connect();
|
||||
|
||||
return connection.getInputStream();
|
||||
}
|
||||
|
||||
static protected void formulateQuery(Set<String> guids, JSONObject node, Writer writer) throws JSONException {
|
||||
JSONWriter jsonWriter = new JSONWriter(writer);
|
||||
|
||||
jsonWriter.object();
|
||||
jsonWriter.key("query");
|
||||
jsonWriter.array();
|
||||
static protected void formulateQuery(Set<String> guids, JSONObject node, Writer writer) throws JSONException {
|
||||
JSONWriter jsonWriter = new JSONWriter(writer);
|
||||
|
||||
jsonWriter.object();
|
||||
|
||||
jsonWriter.key("guid"); jsonWriter.value(null);
|
||||
jsonWriter.key("guid|=");
|
||||
jsonWriter.array();
|
||||
for (String guid : guids) {
|
||||
jsonWriter.value(guid);
|
||||
}
|
||||
jsonWriter.endArray();
|
||||
|
||||
formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
|
||||
|
||||
jsonWriter.endObject();
|
||||
jsonWriter.endArray();
|
||||
jsonWriter.endObject();
|
||||
}
|
||||
|
||||
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
|
||||
String propertyID = node.getString("id");
|
||||
String expectedTypeID = node.getString("expected");
|
||||
|
||||
writer.key(propertyID);
|
||||
writer.array();
|
||||
{
|
||||
if (!expectedTypeID.startsWith("/type/")) { // not literal
|
||||
writer.object();
|
||||
writer.key("limit"); writer.value(10);
|
||||
{
|
||||
boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
|
||||
jsonWriter.key("query");
|
||||
jsonWriter.array();
|
||||
jsonWriter.object();
|
||||
|
||||
jsonWriter.key("guid"); jsonWriter.value(null);
|
||||
jsonWriter.key("guid|=");
|
||||
jsonWriter.array();
|
||||
for (String guid : guids) {
|
||||
jsonWriter.value(guid);
|
||||
}
|
||||
jsonWriter.endArray();
|
||||
|
||||
if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
|
||||
writer.key("name"); writer.value(null);
|
||||
writer.key("id"); writer.value(null);
|
||||
writer.key("guid"); writer.value(null);
|
||||
writer.key("type"); writer.array(); writer.endArray();
|
||||
}
|
||||
|
||||
if (hasSubProperties) {
|
||||
formulateQueryNode(node.getJSONArray("properties"), writer);
|
||||
}
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
|
||||
|
||||
jsonWriter.endObject();
|
||||
jsonWriter.endArray();
|
||||
jsonWriter.endObject();
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
|
||||
static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
|
||||
int l = propertiesA.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
formulateQueryNode(propertiesA.getJSONObject(i), writer);
|
||||
}
|
||||
}
|
||||
|
||||
static protected int countColumns(JSONObject obj, List<String> columnNames) throws JSONException {
|
||||
if (obj.has("properties") && !obj.isNull("properties")) {
|
||||
boolean included = (obj.has("included") && obj.getBoolean("included"));
|
||||
if (included && columnNames != null) {
|
||||
columnNames.add(obj.getString("name"));
|
||||
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
|
||||
String propertyID = node.getString("id");
|
||||
String expectedTypeID = node.getString("expected");
|
||||
|
||||
writer.key(propertyID);
|
||||
writer.array();
|
||||
{
|
||||
if (!expectedTypeID.startsWith("/type/")) { // not literal
|
||||
writer.object();
|
||||
writer.key("limit"); writer.value(10);
|
||||
writer.key("optional"); writer.value(true);
|
||||
{
|
||||
boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
|
||||
|
||||
if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
|
||||
writer.key("name"); writer.value(null);
|
||||
writer.key("id"); writer.value(null);
|
||||
writer.key("guid"); writer.value(null);
|
||||
writer.key("type"); writer.array(); writer.endArray();
|
||||
}
|
||||
|
||||
if (hasSubProperties) {
|
||||
formulateQueryNode(node.getJSONArray("properties"), writer);
|
||||
}
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames);
|
||||
} else {
|
||||
if (columnNames != null) {
|
||||
columnNames.add(obj.getString("name"));
|
||||
writer.endArray();
|
||||
}
|
||||
|
||||
static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
|
||||
int l = propertiesA.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
formulateQueryNode(propertiesA.getJSONObject(i), writer);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static protected int countColumns(JSONArray a, List<String> columnNames) throws JSONException {
|
||||
int c = 0;
|
||||
int l = a.length();
|
||||
for (int i = 0; i < l; i++) {
|
||||
c += countColumns(a.getJSONObject(i), columnNames);
|
||||
|
||||
static protected int countColumns(JSONObject obj, List<String> columnNames) throws JSONException {
|
||||
if (obj.has("properties") && !obj.isNull("properties")) {
|
||||
boolean included = (obj.has("included") && obj.getBoolean("included"));
|
||||
if (included && columnNames != null) {
|
||||
columnNames.add(obj.getString("name"));
|
||||
}
|
||||
return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames);
|
||||
} else {
|
||||
if (columnNames != null) {
|
||||
columnNames.add(obj.getString("name"));
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static protected int countColumns(JSONArray a, List<String> columnNames) throws JSONException {
|
||||
int c = 0;
|
||||
int l = a.length();
|
||||
for (int i = 0; i < l; i++) {
|
||||
c += countColumns(a.getJSONObject(i), columnNames);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
}
|
||||
@ -5,7 +5,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
|
||||
this._extension = { properties: [] };
|
||||
|
||||
var self = this;
|
||||
var frame = DialogSystem.createDialog();
|
||||
var frame = this._frame = DialogSystem.createDialog();
|
||||
frame.width("900px").addClass("extend-data-preview-dialog");
|
||||
|
||||
var header = $('<div></div>').addClass("dialog-header").text("Add Columns from Freebase Based on Column " + column.name).appendTo(frame);
|
||||
@ -25,7 +25,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
|
||||
'<td height="1">Suggested Properties</td>' +
|
||||
'</tr>' +
|
||||
'<tr>' +
|
||||
'<td height="99%"><div class="suggested-property-container"></div></td>' +
|
||||
'<td height="99%"><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>' +
|
||||
'</tr>' +
|
||||
'</table></div>'
|
||||
).appendTo(body);
|
||||
@ -41,16 +41,100 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
|
||||
DialogSystem.dismissUntil(self._level - 1);
|
||||
}).appendTo(footer);
|
||||
|
||||
this._level = DialogSystem.showDialog(frame);
|
||||
ExtendDataPreviewDialog.getAllProperties(column.reconConfig.type.id, function(properties) {
|
||||
self._show(properties);
|
||||
});
|
||||
};
|
||||
|
||||
ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
|
||||
var query = {
|
||||
id : typeID,
|
||||
type : "/type/type",
|
||||
"/freebase/type_hints/included_types" : [{
|
||||
optional: true,
|
||||
properties : [{
|
||||
id : null,
|
||||
name : null,
|
||||
"/type/property/expected_type" : {
|
||||
id : null,
|
||||
"/freebase/type_hints/mediator" : []
|
||||
},
|
||||
sort : "name"
|
||||
}]
|
||||
}],
|
||||
properties : [{
|
||||
id : null,
|
||||
name : null,
|
||||
"/type/property/expected_type" : {
|
||||
id : null,
|
||||
"/freebase/type_hints/mediator" : []
|
||||
},
|
||||
sort : "name"
|
||||
}]
|
||||
};
|
||||
|
||||
var allProperties = [];
|
||||
var processProperty = function(property) {
|
||||
var expectedType = property["/type/property/expected_type"];
|
||||
if (expectedType["/freebase/type_hints/mediator"].length > 0 && expectedType["/freebase/type_hints/mediator"][0]) {
|
||||
|
||||
} else {
|
||||
allProperties.push({
|
||||
id : property.id,
|
||||
name : property.name,
|
||||
expected : expectedType.id
|
||||
});
|
||||
}
|
||||
};
|
||||
var processProperties = function(properties) {
|
||||
$.each(properties, function() { processProperty(this); });
|
||||
};
|
||||
|
||||
$.getJSON(
|
||||
"http://api.freebase.com/api/service/mqlread?query=" + escape(JSON.stringify({ query : query })) + "&callback=?",
|
||||
null,
|
||||
function(o) {
|
||||
if ("result" in o) {
|
||||
processProperties(o.result.properties);
|
||||
$.each(o.result["/freebase/type_hints/included_types"], function() {
|
||||
processProperties(this.properties);
|
||||
})
|
||||
|
||||
onDone(allProperties);
|
||||
} else {
|
||||
onDone([]);
|
||||
}
|
||||
},
|
||||
"jsonp"
|
||||
);
|
||||
};
|
||||
|
||||
ExtendDataPreviewDialog.prototype._show = function(properties) {
|
||||
this._level = DialogSystem.showDialog(this._frame);
|
||||
|
||||
var self = this;
|
||||
var container = this._elmts.suggestedPropertyContainer;
|
||||
var renderSuggestedProperty = function(property) {
|
||||
var div = $('<div>').addClass("suggested-property").appendTo(container);
|
||||
$('<a>').attr("href", "javascript:{}").text(property.name).appendTo(div).click(function() {
|
||||
self._addProperty(property);
|
||||
});
|
||||
};
|
||||
for (var i = 0; i < properties.length; i++) {
|
||||
renderSuggestedProperty(properties[i]);
|
||||
}
|
||||
|
||||
var suggestConfig = {
|
||||
type: '/type/property'
|
||||
type: '/type/property',
|
||||
schema: this._column.reconConfig.type.id
|
||||
};
|
||||
if ("reconConfig" in column) {
|
||||
suggestConfig.schema = column.reconConfig.type.id;
|
||||
}
|
||||
|
||||
this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
|
||||
self._addProperty(data);
|
||||
self._addProperty({
|
||||
id : data.id,
|
||||
name: data.name,
|
||||
expected: data["/type/property/expected_type"]
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
@ -77,11 +161,7 @@ ExtendDataPreviewDialog.prototype._update = function() {
|
||||
};
|
||||
|
||||
ExtendDataPreviewDialog.prototype._addProperty = function(p) {
|
||||
this._extension.properties.push({
|
||||
id : p.id,
|
||||
name: p.name,
|
||||
expected: p["/type/property/expected_type"]
|
||||
});
|
||||
this._extension.properties.push(p);
|
||||
this._update();
|
||||
};
|
||||
|
||||
@ -93,6 +173,12 @@ ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
|
||||
}
|
||||
|
||||
var table = $('<table>')[0];
|
||||
var trHead = table.insertRow(table.rows.length);
|
||||
$(trHead.insertCell(trHead.cells.length)).text(this._column.name);
|
||||
|
||||
for (var c = 0; c < data.columns.length; c++) {
|
||||
$(trHead.insertCell(trHead.cells.length)).text(data.columns[c]);
|
||||
}
|
||||
|
||||
for (var r = 0; r < data.rows.length; r++) {
|
||||
var tr = table.insertRow(table.rows.length);
|
||||
|
||||
@ -819,13 +819,17 @@ DataTableColumnHeaderUI.prototype._doAddColumn = function(initialExpression) {
|
||||
};
|
||||
|
||||
DataTableColumnHeaderUI.prototype._doAddColumnFromFreebase = function() {
|
||||
var o = DataTableView.sampleVisibleRows(this._column);
|
||||
if ("reconConfig" in this._column && "type" in this._column.reconConfig) {
|
||||
var o = DataTableView.sampleVisibleRows(this._column);
|
||||
|
||||
new ExtendDataPreviewDialog(
|
||||
this._column,
|
||||
o.rowIndices,
|
||||
function() {}
|
||||
);
|
||||
new ExtendDataPreviewDialog(
|
||||
this._column,
|
||||
o.rowIndices,
|
||||
function() {}
|
||||
);
|
||||
} else {
|
||||
alert("This column has not been reconciled yet.");
|
||||
}
|
||||
};
|
||||
|
||||
DataTableColumnHeaderUI.prototype._doRemoveColumn = function() {
|
||||
|
||||
@ -1,10 +1,14 @@
|
||||
.extend-data-preview-dialog .suggested-property-container {
|
||||
border: 1px solid #aaa;
|
||||
padding: 10px;
|
||||
padding: 5px;
|
||||
overflow: auto;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.extend-data-preview-dialog .suggested-property {
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
.extend-data-preview-dialog input.property-suggest {
|
||||
display: block;
|
||||
padding: 2%;
|
||||
@ -13,7 +17,15 @@
|
||||
|
||||
.extend-data-preview-dialog .preview-container {
|
||||
border: 1px solid #aaa;
|
||||
padding: 10px;
|
||||
overflow: auto;
|
||||
height: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.extend-data-preview-dialog .preview-container table {
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.extend-data-preview-dialog .preview-container td {
|
||||
padding: 3px 5px;
|
||||
border-bottom: 1px solid #ddd;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user