s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extracted out Freebase related functionalities. Another check-in will remove that code from the core module

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1451 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-10-08 01:39:38 +00:00
parent 5040b06d9f
commit be17d0b3b4
66 changed files with 7771 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
extensions/freebase/.classpath Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry combineaccessrules="false" kind="src" path="/grefine"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry kind="lib" path="/grefine-server/lib/servlet-api-2.5.jar" sourcepath="/grefine-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/grefine-server/lib/slf4j-api-1.5.6.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/httpcore-4.0.1.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/httpcore-4.0.1-sources.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.1.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/signpost-core-1.2.1.1.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
<classpathentry kind="lib" path="/grefine/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="/grefine/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry kind="output" path="module/MOD-INF/classes"/>
</classpath>

17
extensions/freebase/.project Normal file
View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>grefine-freebase</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

71
extensions/freebase/build.xml Normal file
View File

@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--+
|
| Google Refine Extension Freebase Build File
|
+-->
<project name="refine-freebase" default="build" basedir=".">
<property environment="env"/>
<condition property="version" value="trunk">
<not><isset property="version"/></not>
</condition>
<condition property="revision" value="rXXXX">
<not><isset property="revision"/></not>
</condition>
<condition property="full_version" value="0.0.0.0">
<not><isset property="full_version"/></not>
</condition>
<condition property="dist.dir" value="dist">
<not><isset property="dist.dir"/></not>
</condition>
<property name="fullname" value="${name}-${version}-${revision}" />
<property name="refine.dir" value="${basedir}/../../main" />
<property name="refine.webinf.dir" value="${refine.dir}/webapp/WEB-INF" />
<property name="refine.modinf.dir" value="${refine.dir}/webapp/modules/core/MOD-INF" />
<property name="refine.classes.dir" value="${refine.webinf.dir}/classes" />
<property name="refine.lib.dir" value="${refine.webinf.dir}/lib" />
<property name="server.dir" value="${basedir}/../../server" />
<property name="server.lib.dir" value="${server.dir}/lib" />
<property name="src.dir" value="${basedir}/src" />
<property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" />
<property name="classes.dir" value="${modinf.dir}/classes" />
<path id="class.path">
<fileset dir="${lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${refine.lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${server.lib.dir}">
<include name="**/*.jar" />
</fileset>
<pathelement path="${refine.classes.dir}"/>
</path>
<target name="build_java">
<mkdir dir="${classes.dir}" />
<javac destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/>
<classpath refid="class.path" />
</javac>
</target>
<target name="build" depends="build_java"/>
<target name="clean">
<delete dir="${classes.dir}" />
</target>
</project>

79
extensions/freebase/module/MOD-INF/controller.js Normal file
View File

@ -0,0 +1,79 @@
function init() {
Packages.com.google.refine.freebase.oauth.FreebaseProvider.register();
var RS = Packages.com.google.refine.RefineServlet;
RS.registerClassMapping(
"com.google.refine.model.changes.DataExtensionChange",
"com.google.refine.freebase.model.changes.DataExtensionChange");
RS.registerClassMapping(
"com.google.refine.operations.SaveProtographOperation$ProtographChange",
"com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange");
// TODO(dfhuynh): Temporary hack until we know how the core module can resolve our module's classes
RS.cacheClass(Packages.com.google.refine.freebase.model.changes.DataExtensionChange);
RS.cacheClass(Packages.com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange);
RS.registerCommand(module, "extend-data", new Packages.com.google.refine.freebase.commands.ExtendDataCommand());
RS.registerCommand(module, "preview-extend-data", new Packages.com.google.refine.freebase.commands.PreviewExtendDataCommand());
RS.registerCommand(module, "preview-protograph", new Packages.com.google.refine.freebase.commands.PreviewProtographCommand());
RS.registerCommand(module, "save-protograph", new Packages.com.google.refine.freebase.commands.SaveProtographCommand());
RS.registerCommand(module, "check-authorization", new Packages.com.google.refine.freebase.commands.auth.CheckAuthorizationCommand());
RS.registerCommand(module, "authorize", new Packages.com.google.refine.freebase.commands.auth.AuthorizeCommand());
RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.freebase.commands.auth.DeAuthorizeCommand());
RS.registerCommand(module, "user-badges", new Packages.com.google.refine.freebase.commands.auth.GetUserBadgesCommand());
RS.registerCommand(module, "upload-data", new Packages.com.google.refine.freebase.commands.UploadDataCommand());
RS.registerCommand(module, "import-qa-data", new Packages.com.google.refine.freebase.commands.ImportQADataCommand());
RS.registerCommand(module, "mqlread", new Packages.com.google.refine.freebase.commands.MQLReadCommand());
RS.registerCommand(module, "mqlwrite", new Packages.com.google.refine.freebase.commands.MQLWriteCommand());
var OR = Packages.com.google.refine.operations.OperationRegistry;
OR.registerOperation(module, "extend-data", Packages.com.google.refine.freebase.operations.ExtendDataOperation);
OR.registerOperation(module, "import-qa-data", Packages.com.google.refine.freebase.operations.ImportQADataOperation);
OR.registerOperation(module, "save-protograph", Packages.com.google.refine.freebase.operations.SaveProtographOperation); // for backward compatibility
OR.registerOperation(module, "save-schema-alignment-skeleton", Packages.com.google.refine.freebase.operations.SaveProtographOperation);
var RC = Packages.com.google.refine.model.recon.ReconConfig;
RC.registerReconConfig(module, "strict", Packages.com.google.refine.freebase.model.recon.StrictReconConfig);
RC.registerReconConfig(module, "extend", Packages.com.google.refine.freebase.model.recon.DataExtensionReconConfig);
var ER = Packages.com.google.refine.exporters.ExporterRegistry;
ER.registerExporter("tripleloader", new Packages.com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter());
ER.registerExporter("mqlwrite", new Packages.com.google.refine.freebase.ProtographTransposeExporter.MqlwriteLikeExporter());
Packages.com.google.refine.model.Project.
registerOverlayModel("freebaseProtograph", Packages.com.google.refine.freebase.protograph.Protograph);
ClientSideResourceManager.addPaths(
"project/scripts",
module,
[
"scripts/extension.js",
"scripts/util/sign.js",
"scripts/util/freebase.js",
"scripts/dialogs/freebase-loading-dialog.js",
"scripts/dialogs/extend-data-preview-dialog.js",
"scripts/dialogs/schema-alignment/dialog.js",
"scripts/dialogs/schema-alignment/ui-node.js",
"scripts/dialogs/schema-alignment/ui-link.js"
]
);
ClientSideResourceManager.addPaths(
"project/styles",
module,
[
"styles/dialogs/freebase-loading-dialog.less",
"styles/dialogs/extend-data-preview-dialog.less",
"styles/dialogs/schema-alignment-dialog.less"
]
);
}

4
extensions/freebase/module/MOD-INF/module.properties Normal file
View File

@ -0,0 +1,4 @@
name = freebase-extension
description = Google Refine Freebase Extension
templating = false
requires = core

26
extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html Normal file
View File

@ -0,0 +1,26 @@
<div class="dialog-frame extend-data-preview-dialog" style="width: 1000px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div class="grid-layout layout-normal layout-full"><table rows="4">
<tr>
<td width="300" height="1">Add Property</td>
<td height="1">Preview</td>
<td height="1" width="1%"><button bind="resetButton">Reset</button></td>
</tr>
<tr>
<td style="vertical-align: top;" height="1"><div class="input-container"><input bind="addPropertyInput" /></div></td>
<td style="vertical-align: top;" rowspan="3" colspan="2"><div class="preview-container" bind="previewContainer"></div></td>
</tr>
<tr>
<td height="1">Suggested Properties</td>
</tr>
<tr>
<td><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>
</tr>
</table></div>
</div>
<div class="dialog-footer" bind="dialogFooter">
<button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>
<button bind="cancelButton">Cancel</button>
</div>
</div>

363
extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js Normal file
View File

@ -0,0 +1,363 @@
function ExtendDataPreviewDialog(column, columnIndex, rowIndices, onDone) {
this._column = column;
this._columnIndex = columnIndex;
this._rowIndices = rowIndices;
this._onDone = onDone;
this._extension = { properties: [] };
var self = this;
this._dialog = $(DOM.loadHTML("freebase-extension", "scripts/dialogs/extend-data-preview-dialog.html"));
this._elmts = DOM.bind(this._dialog);
this._elmts.dialogHeader.text("Add Columns from Freebase Based on Column " + column.name);
this._elmts.resetButton.click(function() {
self._extension.properties = [];
self._update();
});
this._elmts.okButton.click(function() {
if (self._extension.properties.length === 0) {
alert("Please add some properties first.");
} else {
DialogSystem.dismissUntil(self._level - 1);
self._onDone(self._extension);
}
});
this._elmts.cancelButton.click(function() {
DialogSystem.dismissUntil(self._level - 1);
});
var dismissBusy = DialogSystem.showBusy();
var type = "reconConfig" in column && "type" in column.reconConfig ? column.reconConfig.type.id : "/common/topic";
ExtendDataPreviewDialog.getAllProperties(type, function(properties) {
dismissBusy();
self._show(properties);
});
}
ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
var done = false;
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?type=" + typeID + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
var allProperties = [];
for (var i = 0; i < data.properties.length; i++) {
var property = data.properties[i];
var property2 = {
id: property.id,
name: property.name
};
if ("id2" in property) {
property2.expected = property.schema2;
property2.properties = [{
id: property.id2,
name: property.name2,
expected: property.expects
}];
} else {
property2.expected = property.expects;
}
allProperties.push(property2);
}
allProperties.sort(function(a, b) { return a.name.localeCompare(b.name); });
onDone(allProperties);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
ExtendDataPreviewDialog.prototype._show = function(properties) {
this._level = DialogSystem.showDialog(this._dialog);
var n = this._elmts.suggestedPropertyContainer.offset().top +
this._elmts.suggestedPropertyContainer.outerHeight(true) -
this._elmts.addPropertyInput.offset().top;
this._elmts.previewContainer.height(Math.floor(n));
var self = this;
var container = this._elmts.suggestedPropertyContainer;
var renderSuggestedProperty = function(property) {
var label = ("properties" in property) ? (property.name + " &raquo; " + property.properties[0].name) : property.name;
var div = $('<div>').addClass("suggested-property").appendTo(container);
$('<a>')
.attr("href", "javascript:{}")
.html(label)
.appendTo(div)
.click(function() {
self._addProperty(property);
});
};
for (var i = 0; i < properties.length; i++) {
renderSuggestedProperty(properties[i]);
}
var suggestConfig = {
type: '/type/property'
};
if ("reconConfig" in this._column && "type" in this._column.reconConfig) {
suggestConfig.ac_param = { schema: this._column.reconConfig.type.id };
}
this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
var expected = data.expected_type;
self._addProperty({
id : data.id,
name: data.name,
expected: {
id: expected.id,
name: expected.name
}
});
});
};
ExtendDataPreviewDialog.prototype._update = function() {
this._elmts.previewContainer.empty().text("Querying Freebase ...");
var self = this;
var params = {
project: theProject.id,
columnName: this._column.name
};
$.post(
"/command/freebase-extension/preview-extend-data?" + $.param(params),
{
rowIndices: JSON.stringify(this._rowIndices),
extension: JSON.stringify(this._extension)
},
function(data) {
self._renderPreview(data);
},
"json"
);
};
ExtendDataPreviewDialog.prototype._addProperty = function(p) {
var addSeveralToList = function(properties, oldProperties) {
for (var i = 0; i < properties.length; i++) {
addToList(properties[i], oldProperties);
}
};
var addToList = function(property, oldProperties) {
for (var i = 0; i < oldProperties.length; i++) {
var oldProperty = oldProperties[i];
if (oldProperty.id == property.id) {
if ("included" in property) {
oldProperty.included = "included" in oldProperty ?
(oldProperty.included || property.included) :
property.included;
}
if ("properties" in property) {
if ("properties" in oldProperty) {
addSeveralToList(property.properties, oldProperty.properties);
} else {
oldProperty.properties = property.properties;
}
}
return;
}
}
oldProperties.push(property);
};
addToList(p, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
var self = this;
var container = this._elmts.previewContainer.empty();
if (data.code == "error") {
container.text("Error.");
return;
}
var table = $('<table>')[0];
var trHead = table.insertRow(table.rows.length);
$('<th>').appendTo(trHead).text(this._column.name);
var renderColumnHeader = function(column) {
var th = $('<th>').appendTo(trHead);
$('<span>').html(column.names.join(" &raquo; ")).appendTo(th);
$('<br>').appendTo(th);
$('<a href="javascript:{}"></a>')
.text("remove")
.addClass("action")
.attr("title", "Remove this column")
.click(function() {
self._removeProperty(column.path);
}).appendTo(th);
$('<a href="javascript:{}"></a>')
.text("constrain")
.addClass("action")
.attr("title", "Add constraints to this column")
.click(function() {
self._constrainProperty(column.path);
}).appendTo(th);
};
for (var c = 0; c < data.columns.length; c++) {
renderColumnHeader(data.columns[c]);
}
for (var r = 0; r < data.rows.length; r++) {
var tr = table.insertRow(table.rows.length);
var row = data.rows[r];
for (var c = 0; c < row.length; c++) {
var td = tr.insertCell(tr.cells.length);
var cell = row[c];
if (cell !== null) {
if ($.isPlainObject(cell)) {
$('<a>').attr("href", "http://www.freebase.com/view" + cell.id).text(cell.name).appendTo(td);
} else {
$('<span>').text(cell).appendTo(td);
}
}
}
}
container.append(table);
};
ExtendDataPreviewDialog.prototype._removeProperty = function(path) {
var removeFromList = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
if ("included" in property) {
delete property.included;
}
} else if ("properties" in property && property.properties.length > 0) {
removeFromList(path, index + 1, property.properties);
}
if (!("properties" in property) || property.properties.length === 0) {
properties.splice(i, 1);
}
return;
}
}
};
removeFromList(path, 0, this._extension.properties);
this._update();
};
ExtendDataPreviewDialog.prototype._findProperty = function(path) {
var find = function(path, index, properties) {
var id = path[index];
for (var i = properties.length - 1; i >= 0; i--) {
var property = properties[i];
if (property.id == id) {
if (index === path.length - 1) {
return property;
} else if ("properties" in property && property.properties.length > 0) {
return find(path, index + 1, property.properties);
}
break;
}
}
return null;
};
return find(path, 0, this._extension.properties);
};
ExtendDataPreviewDialog.prototype._constrainProperty = function(path) {
var self = this;
var property = this._findProperty(path);
var frame = DialogSystem.createDialog();
frame.width("500px");
var header = $('<div></div>').addClass("dialog-header").text("Constrain " + path.join(" > ")).appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
body.html(
'<div class="grid-layout layout-normal layout-full"><table>' +
'<tr><td>' +
'Enter MQL query constraints as JSON' +
'</td></tr>' +
'<tr><td>' +
'<textarea style="width: 100%; height: 300px; font-family: monospace;" bind="textarea"></textarea>' +
'</td></tr>' +
'</table></div>'
);
var bodyElmts = DOM.bind(body);
if ("constraints" in property) {
bodyElmts.textarea[0].value = JSON.stringify(property.constraints, null, 2);
} else {
bodyElmts.textarea[0].value = JSON.stringify({ "limit" : 10 }, null, 2);
}
footer.html(
'<button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>' +
'<button bind="cancelButton">Cancel</button>'
);
var footerElmts = DOM.bind(footer);
var level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(level - 1);
};
footerElmts.cancelButton.click(dismiss);
footerElmts.okButton.click(function() {
try {
var o = JSON.parse(bodyElmts.textarea[0].value);
if (o === undefined) {
alert("Please ensure that the JSON you enter is valid.");
return;
}
if ($.isArray(o) && o.length == 1) {
o = o[0];
}
if (!$.isPlainObject(o)) {
alert("The JSON you enter must be an object, that is, it is of this form { ... }.");
return;
}
property.constraints = o;
dismiss();
self._update();
} catch (e) {
//console.log(e);
}
});
bodyElmts.textarea.focus();
};

19
extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html Normal file
View File

@ -0,0 +1,19 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader">Load Data into Freebase</div>
<div class="dialog-body" bind="dialogBody">
</div>
<div class="dialog-footer" bind="dialogFooter">
<table width="100%"><tr>
<td bind="left" style="text-align: left" width="40%" nowrap="true">
<button bind="cancelButton" id="freebase-loading-cancel">Cancel</button>
</td>
<td bind="center" style="text-align: center" width="20%" nowrap="true">
<div bind="authorization" class="freebase-loading-authorization"></div>
</td>
<td bind="right" style="text-align: right" width="40%" nowrap="true">
<span bind="qaCheckboxContainer">Ready to perform QA on this data? <input bind="qaCheckbox" type="checkbox" name="qa" value="qa"></span>
<button bind="loadButton" id="freebase-loading-load" disabled="disabled">Load to Sandbox</button>
</td>
</tr></table>
</div>
</div>

225
extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js Normal file
View File

@ -0,0 +1,225 @@
function FreebaseLoadingDialog() {
this._createDialog();
this._signedin = false;
}
FreebaseLoadingDialog.prototype._createDialog = function() {
var self = this;
var dialog = $(DOM.loadHTML("freebase-extension", "scripts/dialogs/freebase-loading-dialog.html"));
this._elmts = DOM.bind(dialog);
this._elmts.cancelButton.click(function() { self._dismiss(); });
var provider = "www.freebase.com";
var authorization = this._elmts.authorization;
var loadButton = this._elmts.loadButton;
var check_authorization = function(cont) {
$.get("/command/freebase-extension/check-authorization/" + provider, function(data) {
if ("status" in data && data.code == "/api/status/ok") {
authorization.html('Signed in as: <a target="_new" href="http://www.freebase.com/view/user/' + data.username + '">' + data.username + '</a> | <a href="javascript:{}" bind="signout">Sign Out</a>').show();
DOM.bind(authorization).signout.click(function() {
self._signedin = false;
loadButton.attr("disabled","disabled");
$("#freebase-loading-graph-selector-freebase").attr("disabled","disabled").button("refresh");
Sign.signout(check_authorization,provider);
});
loadButton.unbind().click(function() {
self._load();
});
self._signedin = true;
$("#freebase-loading-source-name").keyup();
if (typeof cont == "function") cont(data);
} else {
authorization.html('<a href="javascript:{}" bind="signin">Sign into Freebase</a> to enable loading').show();
DOM.bind(authorization).signin.click(function() {
Sign.signin(function() {
check_authorization(cont);
},provider);
});
}
},"json");
};
var make_topic = function(new_topic_id, topic_type, cont) {
var mql_query = [{
"create": "unless_exists",
"name": new_topic_id,
"a:type": topic_type,
"b:type": "/common/topic",
"id": null,
"guid": null
}];
$.post("/command/freebase-extension/mqlwrite/" + provider,
{ "query" : JSON.stringify(mql_query) },
function(data) {
if ("status" in data && data.code == "/api/status/ok") {
self._elmts.source_id.val(data.result[0].id);
if (typeof cont == "function") cont();
} else {
self._show_error("Error creating new topic", data);
}
},
"json"
);
};
var show_triples = function(cont) {
$.post(
"/command/freebase-extension/preview-protograph?" + $.param({ project: theProject.id }),
{
protograph: JSON.stringify(theProject.overlayModels.freebaseProtograph || {}),
engine: JSON.stringify(ui.browsingEngine.getJSON())
},
function(data) {
var body = self._elmts.dialogBody;
if ("tripleloader" in data) {
body.html(
'<div class="freebase-loading-tripleloader-info"><table><tr>' +
'<td><div>Name this data load &not; <sup style="color: red">required</sup></div>' +
'<input type="text" size="40" id="freebase-loading-source-name" bind="source_name"></td>' +
'<td><div>Source ID &not; <sup style="color: #888">optional</sup></div>' +
'<input type="text" size="60" id="freebase-loading-source-id" bind="source_id"></td>' +
'</tr></table></div>' +
'<div class="freebase-loading-tripleloader-data">' + data.tripleloader + '</div>'
);
self._elmts = DOM.bind(dialog);
self._elmts.source_name.keyup(function() {
if (self._signedin && $(this).val() != "") {
loadButton.removeAttr("disabled");
} else {
loadButton.attr("disabled","disabled");
}
});
self._elmts.source_id.suggest({
"type": "/dataworld/information_source",
"suggest_new": "Click here to add a new information source"
}).bind("fb-select", function(e, data) {
self._elmts.source_id.val(data.id);
}).bind("fb-select-new", function(e, val) {
make_topic(val, "/dataworld/information_source");
});
$.getJSON(
"/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobName" }),
null,
function(data) {
if (data.value != null) {
self._elmts.source_name[0].value = data.value;
}
}
);
if (typeof cont == "function") cont();
} else {
body.html(
'<div class="freebase-loading-tripleloader-message">'+
'<h2>This dataset has no triples</h2>' +
'<p>Have you aligned it with the Freebase schemas yet?</p>' +
'</div>'
);
self._elmts = DOM.bind(dialog);
self._end();
}
self._level = DialogSystem.showDialog(dialog);
},
"json"
);
};
show_triples(check_authorization);
};
FreebaseLoadingDialog.prototype._load = function() {
var self = this;
var qa = self._elmts.qaCheckbox.is(':checked');
var get_refinery_url = function(url) {
return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0];
};
var doLoad = function() {
var dismissBusy = DialogSystem.showBusy();
$.post("/command/freebase-extension/upload-data",
{
project: theProject.id,
"qa" : qa,
"engine" : JSON.stringify(ui.browsingEngine.getJSON()),
"source_name" : self._elmts.source_name.val(),
"source_id" : self._elmts.source_id.val()
},
function(data) {
dismissBusy();
var body = self._elmts.dialogBody;
if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) {
body.html(
'<div class="freebase-loading-tripleloader-message">' +
'<h2><span>' + data.result.added + '</span> triples successfully scheduled for loading</h2>' +
'<h4>Follow the loading progress in the <a href="' + get_refinery_url(data.result.status_url) + '" target="_new">Freebase Refinery</a></h4>' +
'</div>'
);
self._end();
} else {
self._show_error("Error loading data",data);
}
},
"json"
);
};
if (qa) {
var dialog = $(
'<div id="freebase-confirmation-dialog" title="Are you sure?">' +
'<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be tested for upload into <b>Freebase</b>?</td></tr></table>' +
'</div>'
).dialog({
resizable: false,
width: 400,
height: 230,
modal: true,
buttons: {
'yes, I\'m sure': function() {
$(this).dialog('close');
doLoad();
},
'hmm, not really': function() {
$(this).dialog('close');
}
}
});
} else {
doLoad();
}
};
FreebaseLoadingDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
FreebaseLoadingDialog.prototype._show_error = function(msg, error) {
console.log(error);
var self = this;
var body = self._elmts.dialogBody;
body.html(
'<div class="freebase-loading-tripleloader-message">' +
'<h2>' + msg + '</h2>' +
(('message' in error) ? '<p>' + error.message + '</p>' : '<pre>' + JSON.stringify(error, null, 2) + '</pre>') +
(('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</pre>' : "") +
'</div>'
);
this._end();
};
FreebaseLoadingDialog.prototype._end = function() {
var self = this;
self._elmts.loadButton.text("Close").removeAttr("disabled").unbind().click(function() {
self._dismiss();
});
self._elmts.cancelButton.hide();
self._elmts.qaCheckboxContainer.hide();
self._elmts.authorization.hide();
};

290
extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js Normal file
View File

@ -0,0 +1,290 @@
var SchemaAlignment = {};
SchemaAlignment.autoAlign = function() {
var protograph = {};
var columns = theProject.columnModel.columns;
var typedCandidates = [];
var candidates = [];
for (var c = 0; c < columns.length; c++) {
var column = columns[c];
var typed = (column.reconConfig) &&
ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace);
var candidate = {
status: "unbound",
typed: typed,
index: c,
column: column
};
candidates.push(candidate);
if (typed) {
typedCandidates.push(candidate);
}
}
if (typedCandidates.length > 0) {
} else {
var queries = {};
for (var i = 0; i < candidates.length; i++) {
var candidate = candidates[i];
var name = SchemaAlignment._cleanName(candidate.column.name);
var key = "t" + i + ":search";
queries[key] = {
"query" : name,
"limit" : 10,
"type" : "/type/type,/type/property",
"type_strict" : "any"
};
}
SchemaAlignment._batchSearch(queries, function(result) {
console.log(result);
});
}
};
SchemaAlignment._batchSearch = function(queries, onDone) {
var keys = [];
for (var n in queries) {
if (queries.hasOwnProperty(n)) {
keys.push(n);
}
}
var result = {};
var args = [];
var makeBatch = function(keyBatch) {
var batch = {};
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
batch[key] = queries[key];
}
args.push("http://api.freebase.com/api/service/search?" +
$.param({ "queries" : JSON.stringify(batch) }) + "&callback=?");
args.push(null); // no data
args.push(function(data) {
for (var k = 0; k < keyBatch.length; k++) {
var key = keyBatch[k];
result[key] = data[key];
}
});
};
for (var i = 0; i < keys.length; i += 10) {
makeBatch(keys.slice(i, i + 10));
}
args.push(function() {
onDone(result);
});
Ajax.chainGetJSON.apply(null, args);
};
SchemaAlignment._cleanName = function(s) {
return s.replace(/\W/g, " ").replace(/\s+/g, " ").toLowerCase();
};
SchemaAlignment.createNewRootNode = function() {
var rootNode = null;
var links = [];
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
var target = {
nodeType: "cell-as-topic",
columnName: column.name,
createForNoReconMatch: true
};
if ((column.reconConfig) &&
ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) &&
ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace) &&
(column.reconConfig.type)) {
target.type = {
id: column.reconConfig.type.id,
name: column.reconConfig.type.name
};
}
if (column.name == theProject.columnModel.keyColumnName) {
rootNode = target;
} else {
links.push({
property: null,
target: target
});
}
}
rootNode = rootNode || { nodeType: "cell-as-topic" };
rootNode.links = links;
return rootNode;
};
function SchemaAlignmentDialog(protograph, onDone) {
this._onDone = onDone;
this._originalProtograph = protograph || { rootNodes: [] };
this._protograph = cloneDeep(this._originalProtograph); // this is what can be munched on
if (!this._protograph.rootNodes.length) {
this._protograph.rootNodes.push(SchemaAlignment.createNewRootNode());
}
this._nodeUIs = [];
this._createDialog();
this.preview();
}
SchemaAlignmentDialog.prototype._createDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("1000px");
var header = $('<div></div>').addClass("dialog-header").text("Schema Alignment").appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
this._constructFooter(footer);
this._constructBody(body);
this._level = DialogSystem.showDialog(frame);
this._renderBody(body);
};
SchemaAlignmentDialog.prototype._constructFooter = function(footer) {
var self = this;
$('<button></button>').html("&nbsp;&nbsp;OK&nbsp;&nbsp;").click(function() {
var protograph = self.getJSON();
Refine.postProcess(
"freebase-extension",
"save-protograph",
{},
{ protograph: JSON.stringify(protograph) },
{},
{
onDone: function() {
DialogSystem.dismissUntil(self._level - 1);
theProject.overlayModels.freebaseProtograph = protograph;
}
}
);
}).appendTo(footer);
$('<button></button>').text("Cancel").click(function() {
DialogSystem.dismissUntil(self._level - 1);
}).appendTo(footer);
};
SchemaAlignmentDialog.prototype._constructBody = function(body) {
$('<p>' +
'The schema alignment skeleton below specifies how the graph-shaped data that will get generated ' +
'from your grid-shaped data and written into Freebase. The cells in each record of your data will ' +
'get placed into nodes within the skeleton. Configure the skeleton by specifying which ' +
'column to substitute into which node. A node can also be an automatically generated ' +
'anonymous node, or it can be an explicit value or topic that is the same for all records.' +
'</p>').appendTo(body);
$(
'<div id="schema-alignment-tabs" class="refine-tabs">' +
'<ul>' +
'<li><a href="#schema-alignment-tabs-protograph">Skeleton</a></li>' +
'<li><a href="#schema-alignment-tabs-preview-mqllike">MQL-like Preview</a></li>' +
'<li><a href="#schema-alignment-tabs-preview-tripleloader">TripleLoader Preview</a></li>' +
'</ul>' +
'<div id="schema-alignment-tabs-protograph">' +
'<div class="schema-alignment-dialog-canvas"></div>' +
'</div>' +
'<div id="schema-alignment-tabs-preview-mqllike" style="display: none;">' +
'<div class="schema-alignment-dialog-preview"></div>' +
'</div>' +
'<div id="schema-alignment-tabs-preview-tripleloader" style="display: none;">' +
'<div class="schema-alignment-dialog-preview"></div>' +
'</div>' +
'</div>'
).appendTo(body);
};
SchemaAlignmentDialog.prototype._renderBody = function(body) {
var self = this;
$("#schema-alignment-tabs").tabs();
$("#schema-alignment-tabs-preview-mqllike").css("display", "");
$("#schema-alignment-tabs-preview-tripleloader").css("display", "");
this._canvas = $(".schema-alignment-dialog-canvas");
this._nodeTable = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._canvas)[0];
for (var i = 0; i < this._protograph.rootNodes.length; i++) {
this._nodeUIs.push(new SchemaAlignmentDialog.UINode(
this,
this._protograph.rootNodes[i],
this._nodeTable,
{
expanded: true,
mustBeCellTopic: true
}
));
}
this._previewPanes = $(".schema-alignment-dialog-preview");
};
SchemaAlignmentDialog.prototype.getJSON = function() {
var rootNodes = [];
for (var i = 0; i < this._nodeUIs.length; i++) {
var node = this._nodeUIs[i].getJSON();
if (node !== null) {
rootNodes.push(node);
}
}
return {
rootNodes: rootNodes
};
};
SchemaAlignmentDialog.prototype.preview = function() {
var self = this;
this._previewPanes.empty();
var protograph = this.getJSON();
$.post(
"/command/freebase-extension/preview-protograph?" + $.param({ project: theProject.id }),
{ protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
if ("mqllike" in data) {
$(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2));
}
if ("tripleloader" in data) {
$(self._previewPanes[1]).text(data.tripleloader);
}
},
"json"
);
};
SchemaAlignmentDialog._findColumn = function(cellIndex) {
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
if (column.cellIndex == cellIndex) {
return column;
}
}
return null;
};

348
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js Normal file
View File

@ -0,0 +1,348 @@
SchemaAlignmentDialog.UILink = function(dialog, link, table, options, parentUINode) {
this._dialog = dialog;
this._link = link;
this._options = options;
this._parentUINode = parentUINode;
// Make sure target node is there
this._link.target = this._link.target || { nodeType: "cell-as-value" }
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-link-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-link-toggle").attr("width", "1%").addClass("padded");
$(this._tdDetails).addClass("schema-alignment-link-details").attr("width", "90%");
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
var self = this;
var show = function() {
if (self._options.expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._options.expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._options.expanded = !self._options.expanded;
$(this).attr("src", self._options.expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
this._renderMain();
this._renderDetails();
};
SchemaAlignmentDialog.UILink.prototype._renderMain = function() {
$(this._tdMain).empty();
var label = this._link.property !== null ?
(this._link.property.id + ((this._link.condition) ? " [?]" : "")) :
"property?";
var self = this;
$('<img />')
.attr("title", "remove property")
.attr("src", "images/close.png")
.css("cursor", "pointer")
.prependTo(this._tdMain)
.click(function() {
window.setTimeout(function() {
self._parentUINode.removeLink(self);
self._tr.parentNode.removeChild(self._tr);
self._dialog.preview();
}, 100);
});
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-link-tag")
.html(label)
.appendTo(this._tdMain)
.click(function(evt) {
self._startEditProperty(this);
});
$('<img />').attr("src", "images/arrow-start.png").prependTo(a);
$('<img />').attr("src", "images/arrow-end.png").appendTo(a);
};
SchemaAlignmentDialog.UILink.prototype._renderDetails = function() {
if (this._targetUI) {
this._targetUI.dispose();
}
if (this._tableDetails) {
this._tableDetails.remove();
}
this._tableDetails = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv);
this._targetUI = new SchemaAlignmentDialog.UINode(
this._dialog,
this._link.target,
this._tableDetails[0],
{ expanded: "links" in this._link.target && this._link.target.links.length > 0 });
};
SchemaAlignmentDialog.UILink.prototype._startEditProperty = function(elmt) {
var sourceTypeID = this._parentUINode.getExpectedType();
var targetNode = this._targetUI._node;
var targetTypeID = "type" in targetNode && targetNode.type !== null ? targetNode.type.id : null;
var targetTypeName = "columnNames" in targetNode ? targetNode.columnNames[0] : null;
if (sourceTypeID !== null) {
var self = this;
var dismissBusy = DialogSystem.showBusy();
var instanceCount = 0;
var outgoing = [];
var incoming = [];
var onDone = function(properties) {
dismissBusy();
self._showPropertySuggestPopup(
elmt,
properties
);
};
SchemaAlignmentDialog.UILink._getPropertiesOfType(
sourceTypeID,
targetTypeID,
targetTypeName,
onDone
);
} else {
this._showPropertySuggestPopup(elmt, []);
}
};
SchemaAlignmentDialog.UILink._getPropertiesOfType = function(typeID, targetTypeID, targetTypeName, onDone) {
var done = false;
var params = {
"type" : typeID
};
if (targetTypeID !== null) {
params.expects = targetTypeID;
} else if (targetTypeName !== null) {
params.expects = targetTypeName;
}
$.getJSON(
Refine.refineHelperService + "/get_properties_of_type?" + $.param(params) + "&callback=?",
null,
function(data) {
if (done) return;
done = true;
onDone(data.properties || []);
}
);
window.setTimeout(function() {
if (done) return;
done = true;
onDone([]);
}, 7000); // time to give up?
};
SchemaAlignmentDialog.UILink.prototype._showPropertySuggestPopup = function(elmt, suggestions) {
self = this;
var menu = MenuSystem.createMenu().width("350px");
var commitProperty = function(p) {
window.setTimeout(function() { MenuSystem.dismissAll(); }, 100);
if ("id2" in p) {
// self._targetUI.dispose();
self._link.property = {
id: p.id,
name: p.name
};
self._link.target = {
nodeType: "anonymous",
links: [{
property: {
id: p.id2,
name: p.name2
},
target: self._link.target
}]
};
self._renderDetails();
} else {
self._link.property = {
id: p.id,
name: p.name
};
}
var conditionColumnName = conditionalSelect[0].value;
if (conditionColumnName != "") {
self._link.condition = { columnName: conditionColumnName };
} else {
delete self._link.condition;
}
self._configureTarget();
};
var divConditional = $('<div>')
.addClass("schema-alignment-link-menu-section")
.html("Assert link when 'true' is found in column<br/>").appendTo(menu);
var conditionalSelect = $('<select>').appendTo(divConditional);
$('<option>')
.text("(always assert)")
.attr("value", "")
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
for (var c = 0; c < theProject.columnModel.columns.length; c++) {
var column = theProject.columnModel.columns[c];
var option = $('<option>')
.text(column.name)
.attr("value", column.name)
.attr("name", "schema-alignment-link-menu-condition")
.appendTo(conditionalSelect);
if ((self._link.condition) && column.name == self._link.condition.columnName) {
option.attr("selected", "true");
}
}
var divSearch;
if (suggestions.length > 0) {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section")
.html('<div>Search for a property or pick one below</div>').appendTo(menu);
var createSuggestion = function(suggestion) {
var menuItem = MenuSystem.createMenuItem().appendTo(menu);
$('<span>')
.text(suggestion.name)
.attr("title", suggestion.id)
.appendTo(menuItem);
if ("name2" in suggestion) {
$('<span>').html(" &raquo; ").appendTo(menuItem);
$('<span>')
.text(suggestion.name2)
.attr("title", suggestion.id2)
.appendTo(menuItem);
}
menuItem.click(function() {
commitProperty(suggestion);
});
};
for (var i = 0; i < suggestions.length && i < 10; i++) {
createSuggestion(suggestions[i]);
}
} else {
divSearch = $('<div>')
.addClass("schema-alignment-link-menu-section-last")
.html('<div>Search for a property</div>').appendTo(menu);
}
var input = $('<input />').appendTo($('<div>').appendTo(divSearch));
MenuSystem.showMenu(menu, function(){});
MenuSystem.positionMenuAboveBelow(menu, $(elmt));
var suggestOptions = {
type : '/type/property'
};
var sourceTypeID = this._parentUINode.getExpectedType();
if (sourceTypeID !== null) {
suggestOptions.ac_param = { schema: sourceTypeID };
}
input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
input[0].focus();
};
SchemaAlignmentDialog.UILink.prototype.getJSON = function() {
if ("property" in this._link && this._link.property !== null &&
"target" in this._link && this._link.target !== null) {
var targetJSON = this._targetUI.getJSON();
if (targetJSON !== null) {
var json = {
property: cloneDeep(this._link.property),
target: targetJSON
};
if (this._link.condition) {
json.condition = cloneDeep(this._link.condition);
}
return json;
}
}
return null;
};
SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
var self = this;
var dismissBusy = DialogSystem.showBusy();
$.getJSON(
"http://api.freebase.com/api/service/mqlread?query=" + JSON.stringify({
query: {
"id" : this._link.property.id,
"type" : "/type/property",
"expected_type" : {
"id" : null,
"name" : null,
"/freebase/type_hints/mediator" : null
}
}
}) + "&callback=?",
null,
function(o) {
dismissBusy();
if ("result" in o) {
var expected_type = o.result.expected_type;
self._link.target.type = {
id: expected_type.id,
name: expected_type.name
};
if (expected_type["/freebase/type_hints/mediator"] === true) {
self._link.target.nodeType = "anonymous";
} else if (expected_type.id == "/type/key") {
self._link.target.nodeType = "cell-as-key";
} else if (expected_type.id.match(/^\/type\//)) {
self._link.target.nodeType = "cell-as-value";
} else if (!("topic" in self._link.target)) {
self._link.target.nodeType = "cell-as-topic";
self._link.target.createForNoReconMatch = true;
}
self._targetUI.render();
}
self._renderMain();
self._dialog.preview();
},
"jsonp"
);
};

732
extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js Normal file
View File

@ -0,0 +1,732 @@
SchemaAlignmentDialog.UINode = function(dialog, node, table, options) {
this._dialog = dialog;
this._node = node;
this._options = options;
if ("columnName" in this._node) {
this._node.columnNames = [ this._node.columnName ];
delete this._node.columnName;
}
this._linkUIs = [];
this._detailsRendered = false;
this._tr = table.insertRow(table.rows.length);
this._tdMain = this._tr.insertCell(0);
this._tdToggle = this._tr.insertCell(1);
this._tdDetails = this._tr.insertCell(2);
$(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded");
$(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide();
$(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide();
this._renderMain();
this._expanded = options.expanded;
if (this._isExpandable()) {
this._showExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.dispose = function() {
// nothing for now
};
SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) {
for (var i = this._linkUIs.length - 1; i >= 0; i--) {
if (this._linkUIs[i] === linkUI) {
this._linkUIs.splice(i, 1);
this._node.links.splice(i, 1);
break;
}
}
};
SchemaAlignmentDialog.UINode.prototype._isExpandable = function() {
return this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "anonymous" ||
this._node.nodeType == "topic";
};
SchemaAlignmentDialog.UINode.prototype.render = function() {
this._renderMain();
if (this._isExpandable()) {
this._showExpandable();
} else {
this._hideExpandable();
}
};
SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() {
if ("type" in this._node) {
return this._node.type.id;
}
return null;
};
SchemaAlignmentDialog.UINode.prototype._renderMain = function() {
$(this._tdMain).empty();
var self = this;
var a = $('<a href="javascript:{}"></a>')
.addClass("schema-alignment-node-tag")
.appendTo(this._tdMain)
.click(function(evt) {
self._showNodeConfigDialog();
});
if (this._node.nodeType == "cell-as-topic" ||
this._node.nodeType == "cell-as-value" ||
this._node.nodeType == "cell-as-key") {
if ("columnNames" in this._node) {
for (var c = 0; c < this._node.columnNames.length; c++) {
if (c > 0) {
$('<span>').text(", ").appendTo(a);
}
$('<span>')
.text(this._node.columnNames[c])
.addClass("schema-alignment-node-column")
.appendTo(a);
}
$('<span>').text(this._node.columnNames.length > 1 ? " cells" : " cell").appendTo(a);
} else {
a.html(this._options.mustBeCellTopic ? "Which column?" : "Configure...");
}
} else if (this._node.nodeType == "topic") {
if ("topic" in this._node) {
a.html(this._node.topic.name);
} else if ("id" in this._node) {
a.html(this._node.topic.id);
} else {
a.html("Which topic?");
}
} else if (this._node.nodeType == "value") {
if ("value" in this._node) {
a.html(this._node.value);
} else {
a.html("What value?");
}
} else if (this._node.nodeType == "anonymous") {
a.html("(anonymous)");
}
};
SchemaAlignmentDialog.UINode.prototype._showExpandable = function() {
$(this._tdToggle).show();
$(this._tdDetails).show();
if (this._detailsRendered) {
return;
}
this._detailsRendered = true;
this._collapsedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("padded").html("...");
this._expandedDetailDiv = $('<div></div>').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
this._renderDetails();
var self = this;
var show = function() {
if (self._expanded) {
self._collapsedDetailDiv.hide();
self._expandedDetailDiv.show();
} else {
self._collapsedDetailDiv.show();
self._expandedDetailDiv.hide();
}
};
show();
$(this._tdToggle).html("&nbsp;");
$('<img />')
.attr("src", this._expanded ? "images/expanded.png" : "images/collapsed.png")
.appendTo(this._tdToggle)
.click(function() {
self._expanded = !self._expanded;
$(this).attr("src", self._expanded ? "images/expanded.png" : "images/collapsed.png");
show();
});
};
SchemaAlignmentDialog.UINode.prototype._hideExpandable = function() {
$(this._tdToggle).hide();
$(this._tdDetails).hide();
};
SchemaAlignmentDialog.UINode.prototype._renderDetails = function() {
var self = this;
this._tableLinks = $('<table></table>').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv)[0];
if ("links" in this._node && this._node.links !== null) {
for (var i = 0; i < this._node.links.length; i++) {
this._linkUIs.push(new SchemaAlignmentDialog.UILink(
this._dialog,
this._node.links[i],
this._tableLinks,
{ expanded: true },
this
));
}
}
var divFooter = $('<div></div>').addClass("padded").appendTo(this._expandedDetailDiv);
$('<a href="javascript:{}"></a>')
.addClass("action")
.text("add property")
.appendTo(divFooter)
.click(function() {
var newLink = {
property: null,
target: {
nodeType: "cell-as-value"
}
};
self._linkUIs.push(new SchemaAlignmentDialog.UILink(
self._dialog,
newLink,
self._tableLinks,
{
expanded: true,
mustBeCellTopic: false
},
self
));
});
};
SchemaAlignmentDialog.UINode.prototype._showColumnPopupMenu = function(elmt) {
self = this;
var menu = [];
if (!this._options.mustBeCellTopic) {
menu.push({
label: "Anonymous Node",
click: function() {
self._node.nodeType = "anonymous";
self._showExpandable();
self._renderMain();
}
});
menu.push({
label: "Freebase Topic",
click: function() {
self._node.nodeType = "topic";
self._hideExpandable();
self._renderMain();
}
});
menu.push({
label: "Value",
click: function() {
self._node.nodeType = "value";
self._hideExpandable();
self._renderMain();
}
});
menu.push({}); // separator
}
var columns = theProject.columnModel.columns;
var createColumnMenuItem = function(index) {
menu.push({
label: columns[index].name,
click: function() {
self._node.nodeType = "cell-as-topic";
self._node.columnNames = [ columns[index].name ];
self._showExpandable();
self._renderMain();
}
});
};
for (var i = 0; i < columns.length; i++) {
createColumnMenuItem(i);
}
MenuSystem.createAndShowStandardMenu(menu, elmt);
};
SchemaAlignmentDialog.UINode.prototype._showNodeConfigDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("800px");
var header = $('<div></div>').addClass("dialog-header").text("Schema Skeleton Node").appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
/*--------------------------------------------------
* Body
*--------------------------------------------------
*/
var literalTypeSelectHtml =
'<option value="/type/text" checked>text</option>' +
'<option value="/type/int">int</option>' +
'<option value="/type/float">float</option>' +
'<option value="/type/double">double</option>' +
'<option value="/type/boolean">boolean</option>' +
'<option value="/type/datetime">date/time</option>';
var html = $(
'<table class="grid-layout layout-normal layout-full">' +
'<tr>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td>' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="cell-as" bind="radioNodeTypeCellAs" /> Set to Cell in Column' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td><div class="schema-alignment-node-dialog-column-list" bind="divColumns"></div></td>' +
'<td>' +
'<table class="grid-layout layout-tight" cols="4">' +
'<tr>' +
'<td colspan="4">The cell\'s content is used ...</td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-topic" bind="radioNodeTypeCellAsTopic" /></td>' +
'<td colspan="3">to specify a Freebase topic, as reconciled</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="1" width="1%"><input type="checkbox" bind="radioNodeTypeCellAsTopicCreate" /></td>' +
'<td colspan="2">If not reconciled, create new topic named by the cell\'s content, and assign it a type</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td></td>' +
'<td colspan="1">Type:</td>' +
'<td colspan="1"><input bind="cellAsTopicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-value" bind="radioNodeTypeCellAsValue" /></td>' +
'<td colspan="3">as a literal value</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Literal type</td>' +
'<td colspan="1"><select bind="cellAsValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Language (for text)</td>' +
'<td colspan="1"><input bind="cellAsValueLanguageInput" /></td>' +
'</tr>' +
'<tr>' +
'<td><input type="radio" name="schema-align-node-dialog-node-subtype" value="cell-as-key" bind="radioNodeTypeCellAsKey" /></td>' +
'<td colspan="3">as a key in a namespace</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td colspan="2">Namespace</td>' +
'<td colspan="1"><input bind="cellAsKeyInput" /></td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>' +
'</td>' +
'<td>' +
'<table class="grid-layout layout-tight">' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="anonymous" bind="radioNodeTypeAnonymous" /> Generate an anonymous graph node' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Assign a type to the node</td>' +
'<td>&nbsp;<input bind="anonymousNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="topic" bind="radioNodeTypeTopic" /> Use one existing Freebase topic' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Topic</td>' +
'<td><input bind="topicNodeTypeInput" /></td>' +
'</tr>' +
'<tr>' +
'<td colspan="3">' +
'<div class="schema-align-node-dialog-node-type">' +
'<input type="radio" name="schema-align-node-dialog-node-type" value="value" bind="radioNodeTypeValue" /> Use a literal value' +
'</div>' +
'</td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Value</td>' +
'<td><input bind="valueNodeTypeValueInput" /></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Value type</td>' +
'<td><select bind="valueNodeTypeValueTypeSelect">' + literalTypeSelectHtml + '</select></td>' +
'</tr>' +
'<tr>' +
'<td></td>' +
'<td>Language</td>' +
'<td><input bind="valueNodeTypeLanguageInput" /></td>' +
'</tr>' +
'</table>' +
'</td>' +
'</tr>' +
'</table>'
).appendTo(body);
var elmts = DOM.bind(html);
var tableColumns = $('<table></table>')
.attr("cellspacing", "5")
.attr("cellpadding", "0")
.appendTo(elmts.divColumns)[0];
var columnMap = {};
if ("columnNames" in self._node) {
for (var i = 0; i < self._node.columnNames.length; i++) {
columnMap[self._node.columnNames[i]] = true;
}
}
var makeColumnChoice = function(column, columnIndex) {
var tr = tableColumns.insertRow(tableColumns.rows.length);
var radio = $('<input />')
.attr("type", "checkbox")
.attr("value", column.name)
.attr("name", "schema-align-node-dialog-column")
.appendTo(tr.insertCell(0))
.click(function() {
elmts.radioNodeTypeCellAs[0].checked = true;
if ("reconConfig" in column) {
var typeID = column.reconConfig.type.id;
var typeName = column.reconConfig.type.name;
elmts.cellAsTopicNodeTypeInput[0].value = typeName;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", { "id" : typeID, "name" : typeName });
elmts.radioNodeTypeCellAsTopicCreate[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
}
});
if (column.name in columnMap) {
radio.attr("checked", "true");
}
$('<span></span>').text(column.name).appendTo(tr.insertCell(1));
};
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
makeColumnChoice(columns[i], i);
}
elmts.anonymousNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeAnonymous[0].checked = true; })
.suggestT({ type: "/type/type" });
elmts.topicNodeTypeInput
.bind("focus", function() { elmts.radioNodeTypeTopic[0].checked = true; })
.suggest({});
elmts.valueNodeTypeValueInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeValueTypeSelect
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; });
elmts.valueNodeTypeLanguageInput
.bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; })
.suggest({ type: "/type/lang" });
elmts.radioNodeTypeCellAsTopicCreate
.click(function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
});
elmts.cellAsTopicNodeTypeInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsTopic[0].checked = true;
})
.suggestT({ type: "/type/type" });
elmts.cellAsValueTypeSelect
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
});
elmts.cellAsValueLanguageInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsValue[0].checked = true;
})
.suggest({ type: "/type/lang" });
elmts.cellAsKeyInput
.bind("focus", function() {
elmts.radioNodeTypeCellAs[0].checked = true;
elmts.radioNodeTypeCellAsKey[0].checked = true;
})
.suggest({ type: "/type/namespace" });
elmts.radioNodeTypeCellAsTopic[0].checked = true; // just make sure some subtype is selected
if (this._node.nodeType.match(/^cell-as-/)) {
elmts.radioNodeTypeCellAs[0].checked = true;
if (this._node.nodeType == "cell-as-topic") {
elmts.radioNodeTypeCellAsTopic[0].checked = true;
} else if (this._node.nodeType == "cell-as-value") {
elmts.radioNodeTypeCellAsValue[0].checked = true;
} else if (this._node.nodeType == "cell-as-key") {
elmts.radioNodeTypeCellAsKey[0].checked = true;
}
} else if (this._node.nodeType == "anonymous") {
elmts.radioNodeTypeAnonymous[0].checked = true;
} else if (this._node.nodeType == "topic") {
elmts.radioNodeTypeTopic[0].checked = true;
} else if (this._node.nodeType == "value") {
elmts.radioNodeTypeValue[0].checked = true;
}
if ("type" in this._node) {
elmts.anonymousNodeTypeInput[0].value = this._node.type.name;
elmts.anonymousNodeTypeInput.data("data.suggest", this._node.type);
elmts.cellAsTopicNodeTypeInput[0].value = this._node.type.name;
elmts.cellAsTopicNodeTypeInput.data("data.suggest", this._node.type);
}
if ("topic" in this._node) {
elmts.topicNodeTypeInput[0].value = this._node.topic.name;
elmts.topicNodeTypeInput.data("data.suggest", this._node.topic);
}
if ("namespace" in this._node) {
elmts.cellAsKeyInput[0].value = this._node.namespace.name;
elmts.cellAsKeyInput.data("data.suggest", this._node.namespace);
}
if ("createForNoReconMatch" in this._node) {
elmts.radioNodeTypeCellAsTopicCreate[0].checked = this._node.createForNoReconMatch;
}
if ("lang" in this._node) {
elmts.valueNodeTypeLanguageInput[0].value = this._node.lang;
elmts.valueNodeTypeLanguageInput.data("data.suggest", { id: this._node.lang });
elmts.cellAsValueLanguageInput[0].value = this._node.lang;
elmts.cellAsValueLanguageInput.data("data.suggest", { id: this._node.lang });
}
if ("valueType" in this._node) {
elmts.valueNodeTypeValueTypeSelect[0].value = this._node.valueType;
elmts.cellAsValueTypeSelect[0].value = this._node.valueType;
}
/*--------------------------------------------------
* Footer
*--------------------------------------------------
*/
var getResultJSON = function() {
var node = {
nodeType: $("input[name='schema-align-node-dialog-node-type']:checked")[0].value
};
if (node.nodeType == "cell-as") {
node.nodeType = $("input[name='schema-align-node-dialog-node-subtype']:checked")[0].value;
node.columnNames = $("input[name='schema-align-node-dialog-column']:checked").map(function() {
return this.getAttribute("value");
}).get();
if (node.columnNames.length == 0) {
alert("You must select at least one column.");
return null;
}
if (node.nodeType == "cell-as-topic") {
node.createForNoReconMatch = elmts.radioNodeTypeCellAsTopicCreate[0].checked;
var t = elmts.cellAsTopicNodeTypeInput.data("data.suggest");
if (!(t) && node.createForNoReconMatch) {
alert("For creating a new graph node, you need to specify a type for it.");
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "cell-as-value") {
node.valueType = elmts.cellAsValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.cellAsValueLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
} else if (node.nodeType == "cell-as-key") {
var t = elmts.cellAsKeyInput.data("data.suggest");
if (!(t)) {
alert("Please specify the namespace.");
return null;
}
node.namespace = {
id: t.id,
name: t.name
};
}
} else if (node.nodeType == "anonymous") {
var t = elmts.anonymousNodeTypeInput.data("data.suggest");
if (!(t)) {
alert("For generating an anonymous graph node, you need to specify a type for it.");
return null;
}
node.type = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "topic") {
var t = elmts.topicNodeTypeInput.data("data.suggest");
if (!(t)) {
alert("Please specify which existing Freebase topic to use.");
return null;
}
node.topic = {
id: t.id,
name: t.name
};
} else if (node.nodeType == "value") {
node.value = $.trim(elmts.valueNodeTypeValueInput[0].value);
if (!node.value.length) {
alert("Please specify the value to use.");
return null;
}
node.valueType = elmts.valueNodeTypeValueTypeSelect[0].value;
if (node.valueType == "/type/text") {
var l = elmts.valueNodeTypeLanguageInput.data("data.suggest");
node.lang = (l) ? l.id : "/lang/en";
}
}
return node;
};
$('<button></button>').html("&nbsp;&nbsp;OK&nbsp;&nbsp;").click(function() {
var node = getResultJSON();
if (node !== null) {
DialogSystem.dismissUntil(level - 1);
self._node = node;
self.render();
self._dialog.preview();
}
}).appendTo(footer);
$('<button></button>').text("Cancel").click(function() {
DialogSystem.dismissUntil(level - 1);
}).appendTo(footer);
var level = DialogSystem.showDialog(frame);
};
SchemaAlignmentDialog.UINode.prototype.getJSON = function() {
var result = null;
var getLinks = false;
if (this._node.nodeType.match(/^cell-as-/)) {
if (!("columnNames" in this._node) || !this._node.columnNames) {
return null;
}
if (this._node.nodeType == "cell-as-topic") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
type: "type" in this._node ? cloneDeep(this._node.type) : { "id" : "/common/topic", "name" : "Topic", "cvt" : false },
createForNoReconMatch: "createForNoReconMatch" in this._node ? this._node.createForNoReconMatch : true
};
getLinks = true;
} else if (this._node.nodeType == "cell-as-value") {
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "cell-as-key") {
if (!("namespace" in this._node) || !this._node.namespace) {
return null;
}
result = {
nodeType: this._node.nodeType,
columnNames: this._node.columnNames,
namespace: cloneDeep(this._node.namespace)
};
}
} else if (this._node.nodeType == "topic") {
if (!("topic" in this._node) || !this._node.topic) {
return null;
}
result = {
nodeType: this._node.nodeType,
topic: cloneDeep(this._node.topic)
};
getLinks = true;
} else if (this._node.nodeType == "value") {
if (!("value" in this._node) || !this._node.value) {
return null;
}
result = {
nodeType: this._node.nodeType,
value: this._node.value,
valueType: "valueType" in this._node ? this._node.valueType : "/type/text",
lang: "lang" in this._node ? this._node.lang : "/lang/en"
};
} else if (this._node.nodeType == "anonymous") {
if (!("type" in this._node) || !this._node.type) {
return null;
}
result = {
nodeType: this._node.nodeType,
type: cloneDeep(this._node.type)
};
getLinks = true;
}
if (!result) {
return null;
}
if (getLinks) {
var links = [];
for (var i = 0; i < this._linkUIs.length; i++) {
var link = this._linkUIs[i].getJSON();
if (link !== null) {
links.push(link);
}
}
result.links = links;
}
return result;
};

118
extensions/freebase/module/scripts/extension.js Normal file
View File

@ -0,0 +1,118 @@
var FreebaseExtension = { handlers: {} };
FreebaseExtension.handlers.editSchemaAlignment = function(reset) {
new SchemaAlignmentDialog(
reset ? null : theProject.overlayModels.freebaseProtograph, function(newProtograph) {});
};
FreebaseExtension.handlers.loadIntoFreebase = function() {
new FreebaseLoadingDialog();
};
FreebaseExtension.handlers.browseToDataLoad = function() {
// The form has to be created as part of the click handler. If you create it
// inside the getJSON success handler, it won't work.
var form = document.createElement("form");
$(form)
.css("display", "none")
.attr("method", "GET")
.attr("target", "dataload");
document.body.appendChild(form);
var w = window.open("about:blank", "dataload");
$.getJSON(
"/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobID" }),
null,
function(data) {
if (data.value == null) {
alert("You have not tried to load the data in this project into Freebase yet.");
} else {
$(form).attr("action", "http://refinery.freebaseapps.com/load/" + data.value);
form.submit();
w.focus();
}
document.body.removeChild(form);
}
);
};
FreebaseExtension.handlers.importQAData = function() {
Refine.postProcess(
"freebase-extension",
"import-qa-data",
{},
{},
{ cellsChanged: true }
);
};
ExtensionBar.addExtensionMenu({
"id" : "freebase",
"label" : "Freebase",
"submenu" : [
{
"id" : "freebase/schema-alignment",
label: "Edit Schema Aligment Skeleton ...",
click: function() { FreebaseExtension.handlers.editSchemaAlignment(false); }
},
{
"id" : "freebase/reset-schema-alignment",
label: "Reset Schema Alignment Skeleton ...",
click: function() { FreebaseExtension.handlers.editSchemaAlignment(true); }
},
{},
{
"id" : "freebase/load-info-freebase",
label: "Load into Freebase ...",
click: function() { FreebaseExtension.handlers.loadIntoFreebase(); }
},
{
"id" : "freebase/browse-load",
label: "Browse to Data Load ...",
click: function() { FreebaseExtension.handlers.browseToDataLoad(); }
},
{
"id" : "freebase/import-qa-data",
label: "Import QA Data",
click: function() { FreebaseExtension.handlers.importQAData(); }
}
]
});
DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
var columnIndex = Refine.columnNameToColumnIndex(column.name);
var doAddColumnFromFreebase = function() {
var o = DataTableView.sampleVisibleRows(column);
new ExtendDataPreviewDialog(
column,
columnIndex,
o.rowIndices,
function(extension) {
Refine.postProcess(
"freebase-extension",
"extend-data",
{
baseColumnName: column.name,
columnInsertIndex: columnIndex + 1
},
{
extension: JSON.stringify(extension)
},
{ rowsChanged: true, modelsChanged: true }
);
}
);
};
MenuSystem.insertAfter(
menu,
[ "core/edit-column", "core/add-column-by-fetching-urls" ],
{
id: "freebase/add-columns-from-freebase",
label: "Add Columns From Freebase ...",
click: doAddColumnFromFreebase
}
);
});

30
extensions/freebase/module/scripts/util/freebase.js Normal file
View File

@ -0,0 +1,30 @@
var Freebase = {};
Freebase.mqlread = function(query, options, onDone) {
var params = {};
var queryEnv = {
"query": query
};
if (options) {
for (var n in options) {
if (options.hasOwnProperty(n)) {
var v = options[n];
if (typeof v != "string") {
v = JSON.stringify(v);
}
queryEnv[n] = v;
}
}
}
params.query = JSON.stringify(queryEnv);
$.getJSON(
"http://api.freebase.com/api/service/mqlread?" + $.param(params) + "&callback=?",
null,
onDone,
"jsonp"
);
};

120
extensions/freebase/module/scripts/util/sign.js Normal file
View File

@ -0,0 +1,120 @@
if (typeof window.Sign == 'undefined') {
window.Sign = {
window_position: function() {
var position = {};
if (typeof(window.innerWidth) == 'number') {
// Non-IE
position.width = window.outerWidth;
position.height = window.outerHeight;
position.top = window.screenY;
position.left = window.screenX;
} else if (document.documentElement && (document.documentElement.clientWidth || document.documentElement.clientHeight)) {
// IE 6+ in 'standards compliant mode'
position.width = document.body.clientWidth;
position.height = document.body.clientHeight;
position.top = window.screenTop;
position.left = window.screenLeft;
}
return position;
},
popup : function(url, width, height, windowname) {
width = width || 700;
height = height || 500;
var pos = window.Sign.window_position();
var left = Math.floor((pos.width - width) / 2) + pos.left;
var top = Math.floor((pos.height - height) / 2) + pos.top;
// Chrome might fix this bug, but until then add some padding
// to the height of the popup for the urlbar
var is_chrome = /chrome/.test(navigator.userAgent.toLowerCase());
if (is_chrome) {
height += 50;
}
var params = {
width: width,
height: height,
top: top,
left: left,
directories: 'no',
location: 'no',
menubar: 'no',
resizable: 'no',
scrollbars: 'yes',
status: 'no',
toolbar: 'no'
};
var params_list = [];
for (var key in params) {
if (params.hasOwnProperty(key)) {
params_list.push(key + "=" + params[key]);
}
}
return window.open(url, windowname || "", params_list.join(","));
},
signintize : function(cont) {
$('.signedin').show();
$('.signedout').hide();
if (window.user) {
$('.user').html('<a href="http://freebase.com/view' + window.user.id + '">' + window.user.username + '</a>');
}
if (typeof cont == 'function') cont();
},
signin : function(success, provider, width, height) {
var newwin = window.Sign.popup("/command/freebase-extension/authorize/" + provider, width, height);
if (newwin !== null) {
newwin.opener = window;
}
window.onauthorization = function() {
if (typeof success == 'undefined') {
window.location.reload();
} else {
$.ajax({
url: "/command/freebase-extension/check-authorization/" + provider,
dataType: "json",
success: function(data) {
window.user = data;
window.Sign.signintize(success);
}
});
}
};
if (window.focus && newwin !== null) {
newwin.focus();
}
return false;
},
signoutize : function(cont) {
$('.signedin').hide();
$('.signedout').show();
if (typeof cont == 'function') cont();
},
signout : function(success,provider) {
$.ajax({
url: "/command/freebase-extension/deauthorize/" + provider,
success: function() {
if (typeof success == 'undefined') {
window.location.reload();
} else {
window.Sign.signoutize(success);
}
}
});
}
};
}

38
extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less Normal file
View File

@ -0,0 +1,38 @@
@import-less url("../theme.less");
.extend-data-preview-dialog .suggested-property-container {
border: 1px solid #aaa;
padding: 5px;
overflow: auto;
height: 400px;
}
.extend-data-preview-dialog .suggested-property {
padding: 5px;
}
.extend-data-preview-dialog input.property-suggest {
display: block;
padding: 2%;
width: 96%;
}
.extend-data-preview-dialog .preview-container {
border: 1px solid #aaa;
overflow: auto;
}
.extend-data-preview-dialog .preview-container table {
border-collapse: collapse;
}
.extend-data-preview-dialog .preview-container td, .extend-data-preview-dialog .preview-container th {
padding: 3px 5px;
border-bottom: 1px solid #ddd;
border-right: 1px solid #ddd;
}
.extend-data-preview-dialog .preview-container th img {
vertical-align: top;
margin-left: 5px;
}

60
extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less Normal file
View File

@ -0,0 +1,60 @@
@import-less url("../theme.less");
.freebase-loading-tripleloader-data {
height: 400px;
width: 99%;
overflow: auto;
border: 1px solid #aaa;
white-space: pre;
padding: 0.3em 0.5em 0.5em 0.5em;
font-family: monospace;
}
.freebase-loading-tripleloader-message {
height: 500px;
overflow: auto;
border: 1px solid #aaa;
padding: 0.3em 0.5em 0.5em 0.5em;
}
.freebase-loading-tripleloader-message h2 {
text-align: center;
font-size: 150%;
margin-top: 3em;
font-weight: normal;
}
.freebase-loading-tripleloader-message h2 span {
font-size; 130%;
font-weight: bold;
}
.freebase-loading-tripleloader-message h4 {
text-align: center;
font-size: 120%;
margin-top: 2em;
font-weight: normal;
}
.freebase-loading-tripleloader-message a {
font-size: 120%;
font-weight: bold;
}
.freebase-loading-authorization {
margin: 0em 2em;
padding: 0.6em 0.8em;
border: 1px solid #ccc;
background-color: #fff;
-moz-border-radius: 0.8em;
-webkit-border-radius: 0.8em;
}
.freebase-loading-tripleloader-info {
margin-bottom: 0.5em;
}
.freebase-loading-tripleloader-info textarea {
width: 99%;
height: 5em;
}

114
extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less Normal file
View File

@ -0,0 +1,114 @@
@import-less url("../theme.less");
.schema-alignment-dialog-canvas {
height: 400px;
overflow: auto;
border: 1px solid #aaa;
padding: 10px;
background: white;
}
table.schema-alignment-table-layout {
border-collapse: collapse;
margin: 0px;
padding: 0px;
margin-bottom: 1em;
}
table.schema-alignment-table-layout .padded {
padding: 3px 6px;
}
div.schema-alignment-detail-container {
border-left: 3px solid #eee;
}
td.schema-alignment-node-main, td.schema-alignment-link-main {
white-space: pre;
width: 300px;
}
td.schema-alignment-node-toggle, td.schema-alignment-link-toggle {
white-space: pre;
width: 1%;
}
td.schema-alignment-node-toggle img, td.schema-alignment-link-toggle img {
cursor: pointer;
vertical-align: middle;
padding: 2px;
}
td.schema-alignment-node-details, td.schema-alignment-link-details {
width: 90%;
}
a.schema-alignment-node-tag {
padding: 3px 6px;
background: #ddd;
text-decoration: none;
color: black;
-moz-border-radius: 10px;
}
a.schema-alignment-node-tag:hover {
background: #888;
color: white;
}
.schema-alignment-node-column {
font-weight: bold;
}
a.schema-alignment-link-tag {
padding: 3px 6px;
text-decoration: none;
color: black;
}
a.schema-alignment-link-tag:hover {
color: #88f;
}
div.schema-alignment-dialog-preview {
height: 400px;
overflow: auto;
border: 1px solid #aaa;
background: white;
padding: 10px;
white-space: pre;
font-family: monospace;
font-size: 9pt;
}
/*--------------------------------------------------
* Node dialog
*--------------------------------------------------
*/
.schema-align-node-dialog-node-type {
padding: 0.25em;
background: #ddf;
-moz-border-radius: 0.5em;
}
.schema-align-node-dialog-node-type input {
vertical-align: text-bottom;
}
.schema-alignment-node-dialog-column-list {
height: 300px;
width: 200px;
overflow: auto;
border: 1px solid #ddd;
}
/*--------------------------------------------------
* Link dialog
*--------------------------------------------------
*/
.schema-alignment-link-menu-section {
padding: 8px;
border-bottom: 1px solid #ddd;
margin-bottom: 3px;
}
.schema-alignment-link-menu-section-last {
padding: 8px;
}

1
extensions/freebase/module/styles/theme.less Normal file
View File

@ -0,0 +1 @@
@import-less url("../../../../main/webapp/modules/core/styles/theme.less");

9
extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java Normal file
View File

@ -0,0 +1,9 @@
package com.google.refine.freebase;
public class FreebaseProperty extends FreebaseTopic {
//final protected FreebaseType _expectedType;
public FreebaseProperty(String id, String name) {
super(id, name);
}
}

28
extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java Normal file
View File

@ -0,0 +1,28 @@
package com.google.refine.freebase;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
public class FreebaseTopic implements Jsonizable {
final public String id;
final public String name;
public FreebaseTopic(String id, String name) {
this.id = id;
this.name = name;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.endObject();
}
}

36
extensions/freebase/src/com/google/refine/freebase/FreebaseType.java Normal file
View File

@ -0,0 +1,36 @@
package com.google.refine.freebase;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
public class FreebaseType extends FreebaseTopic implements Jsonizable {
public FreebaseType(String id, String name) {
super(id, name);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.endObject();
}
static public FreebaseType load(JSONObject obj) throws Exception {
if (obj == null) {
return null;
}
FreebaseType type = new FreebaseType(
obj.getString("id"),
obj.getString("name")
);
return type;
}
}

75
extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java Normal file
View File

@ -0,0 +1,75 @@
package com.google.refine.freebase;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Properties;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.Exporter;
import com.google.refine.model.Project;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.TransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.Transposer;
import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
abstract public class ProtographTransposeExporter implements Exporter {
final protected String _contentType;
public ProtographTransposeExporter(String contentType) {
_contentType = contentType;
}
public String getContentType() {
return "application/x-unknown";
}
public boolean takeWriter() {
return true;
}
public void export(Project project, Properties options, Engine engine,
OutputStream outputStream) throws IOException {
throw new RuntimeException("Not implemented");
}
public void export(Project project, Properties options, Engine engine,
Writer writer) throws IOException {
Protograph protograph = (Protograph) project.overlayModels.get("freebaseProtograph");
if (protograph != null) {
TransposedNodeFactory nodeFactory = createNodeFactory(project, writer);
Transposer.transpose(project, engine.getAllFilteredRows(),
protograph, protograph.getRootNode(0), nodeFactory, -1);
nodeFactory.flush();
}
}
abstract protected TransposedNodeFactory createNodeFactory(Project project, Writer writer);
static public class TripleLoaderExporter extends ProtographTransposeExporter {
public TripleLoaderExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new TripleLoaderTransposedNodeFactory(project, writer);
}
}
static public class MqlwriteLikeExporter extends ProtographTransposeExporter {
public MqlwriteLikeExporter() {
super("application/x-unknown");
}
@Override
protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
return new MqlwriteLikeTransposedNodeFactory(writer);
}
}
}

32
extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java Normal file
View File

@ -0,0 +1,32 @@
package com.google.refine.freebase.commands;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.freebase.operations.ExtendDataOperation;
import com.google.refine.util.ParsingUtilities;
public class ExtendDataCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project,
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String baseColumnName = request.getParameter("baseColumnName");
int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
String jsonString = request.getParameter("extension");
JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
return new ExtendDataOperation(
engineConfig,
baseColumnName,
extension,
columnInsertIndex
);
}
}

36
extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java Normal file
View File

@ -0,0 +1,36 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.freebase.operations.ImportQADataOperation;
import com.google.refine.process.Process;
public class ImportQADataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
AbstractOperation op = new ImportQADataOperation();
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

31
extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java Normal file
View File

@ -0,0 +1,31 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class MQLReadCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String query = request.getParameter("query");
String result = FreebaseUtils.mqlread(provider,query);
response.getWriter().write(result);
} catch (Exception e) {
respondException(response, e);
}
}
}

40
extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java Normal file
View File

@ -0,0 +1,40 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class MQLWriteCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
if (access_credentials != null) {
String query = request.getParameter("query");
String result = FreebaseUtils.mqlwrite(access_credentials, provider, query);
response.getWriter().write(result);
} else {
respond(response, "401 Unauthorized", "You don't have the right credentials");
}
} catch (Exception e) {
respondException(response, e);
}
}
}

158
extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java Normal file
View File

@ -0,0 +1,158 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseDataExtensionJob;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
public class PreviewExtendDataCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String columnName = request.getParameter("columnName");
String rowIndicesString = request.getParameter("rowIndices");
if (rowIndicesString == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }");
return;
}
String jsonString = request.getParameter("extension");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
int length = rowIndices.length();
int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
List<String> topicNames = new ArrayList<String>();
List<String> topicIds = new ArrayList<String>();
Set<String> ids = new HashSet<String>();
for (int i = 0; i < length; i++) {
int rowIndex = rowIndices.getInt(i);
if (rowIndex >= 0 && rowIndex < project.rows.size()) {
Row row = project.rows.get(rowIndex);
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
topicNames.add(cell.recon.match.name);
topicIds.add(cell.recon.match.id);
ids.add(cell.recon.match.id);
} else {
topicNames.add(null);
topicIds.add(null);
ids.add(null);
}
}
}
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("code"); writer.value("ok");
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.key("names");
writer.array();
for (String name : info.names) {
writer.value(name);
}
writer.endArray();
writer.key("path");
writer.array();
for (String id : info.path) {
writer.value(id);
}
writer.endArray();
writer.endObject();
}
writer.endArray();
writer.key("rows");
writer.array();
for (int r = 0; r < topicNames.size(); r++) {
String id = topicIds.get(r);
String topicName = topicNames.get(r);
if (id != null && map.containsKey(id)) {
DataExtension ext = map.get(id);
boolean first = true;
if (ext.data.length > 0) {
for (Object[] row : ext.data) {
writer.array();
if (first) {
writer.value(topicName);
first = false;
} else {
writer.value(null);
}
for (Object cell : row) {
if (cell != null && cell instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) cell;
writer.object();
writer.key("id"); writer.value(rc.id);
writer.key("name"); writer.value(rc.name);
writer.endObject();
} else {
writer.value(cell);
}
}
writer.endArray();
}
continue;
}
}
writer.array();
if (id != null) {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(topicName);
writer.endObject();
} else {
writer.value("<not reconciled>");
}
writer.endArray();
}
writer.endArray();
writer.endObject();
} catch (Exception e) {
respondException(response, e);
}
}
}

71
extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java Normal file
View File

@ -0,0 +1,71 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.commands.Command;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.model.Project;
import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
import com.google.refine.freebase.protograph.transpose.Transposer;
import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
import com.google.refine.util.ParsingUtilities;
public class PreviewProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
FilteredRows filteredRows = engine.getAllFilteredRows();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
StringBuffer sb = new StringBuffer(2048);
sb.append("{ ");
{
StringWriter stringWriter = new StringWriter();
TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append("\"tripleloader\" : ");
sb.append(JSONObject.quote(stringWriter.toString()));
}
{
StringWriter stringWriter = new StringWriter();
MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter);
Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
nodeFactory.flush();
sb.append(", \"mqllike\" : ");
sb.append(stringWriter.toString());
}
sb.append(" }");
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

40
extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java Normal file
View File

@ -0,0 +1,40 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.google.refine.commands.Command;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.freebase.operations.SaveProtographOperation;
import com.google.refine.process.Process;
import com.google.refine.util.ParsingUtilities;
public class SaveProtographCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("protograph");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
Protograph protograph = Protograph.reconstruct(json);
AbstractOperation op = new SaveProtographOperation(protograph);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (Exception e) {
respondException(response, e);
}
}
}

89
extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java Normal file
View File

@ -0,0 +1,89 @@
package com.google.refine.freebase.commands;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
public class UploadDataCommand extends Command {
final static public String s_dataLoadJobIDPref = "freebase.load.jobID";
final static public String s_dataLoadJobNamePref = "freebase.load.jobName";
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore();
TripleLoaderExporter exporter = new TripleLoaderExporter();
StringWriter triples = new StringWriter(10 * 1024 * 1024);
exporter.export(project, new Properties(), engine, triples);
String source_name = request.getParameter("source_name");
String source_id = request.getParameter("source_id");
String qa = request.getParameter("qa");
String mdo_id = null;
preferenceStore.put(s_dataLoadJobNamePref, source_name);
try {
Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref);
if (jobID != null) {
URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID);
String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
if (!s.equals("null")) {
mdo_id = s;
}
}
} catch (Exception e) {
// ignore
}
String uploadResponse = FreebaseUtils.uploadTriples(
request, qa, source_name, source_id, mdo_id, triples.toString()
);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
try {
JSONObject obj = new JSONObject(uploadResponse);
if (obj.has("result") && !obj.isNull("result")) {
JSONObject result = obj.getJSONObject("result");
if (result.has("job_id") && !result.isNull("job_id")) {
Integer jobID = result.getInt("job_id");
project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
}
}
response.getWriter().write(uploadResponse);
} catch (JSONException e) {
respond(response,"500 Error", uploadResponse);
}
} catch (Exception e) {
respondException(response, e);
} finally {
ProjectManager.singleton.setBusy(false);
}
}
}

135
extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java Normal file
View File

@ -0,0 +1,135 @@
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.OAuthProvider;
import com.google.refine.commands.Command;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class AuthorizeCommand extends Command {
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
// get the provider from the request
Provider provider = OAuthUtilities.getProvider(request);
try {
// see if the request comes with access credentials
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
// prepare the continuation URL that the OAuth provider will redirect the user to
// (we need to make sure this URL points back to this code or the dance will never complete)
String callbackURL = getBaseURL(request,provider);
if (access_credentials == null) {
// access credentials are not available so we need to check
// to see at what stage of the OAuth dance we are
// get the request token credentials
Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST);
OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider);
OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider);
if (request_credentials == null) {
// no credentials were found, so let's start the dance
// get the request token
String url = pp.retrieveRequestToken(consumer, callbackURL);
request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
// and set them to that we can retrieve them later in the second part of the dance
Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600);
// now redirect the user to the Authorize URL where she can authenticate against the
// service provider and authorize us.
// The provider will bounce the user back here for us to continue the dance.
response.sendRedirect(url);
} else {
// we are at the second stage of the dance, so we need need to obtain the access credentials now
// if we got here, it means that the user performed a valid authentication against the
// service provider and authorized us, so now we can request more permanent credentials
// to the service provider and save those as well for later use.
// this is set only for OAuth 1.0a
String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM);
pp.retrieveAccessToken(consumer, verificationCode);
access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
// no matter the result, we need to remove the request token
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600);
finish(response);
}
} else {
finish(response);
}
} catch (Exception e) {
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
respondException(response, e);
}
}
private void finish(HttpServletResponse response) throws IOException {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "text/html");
PrintWriter writer = response.getWriter();
writer.write(
"<html>" +
"<body></body>" +
"<script type='text/javascript'>" +
"if (top.opener && top.opener.onauthorization) {" +
" top.opener.onauthorization(window);" +
"}" +
"self.close();" +
"</script>" +
"</html>"
);
writer.flush();
}
private String getBaseURL(HttpServletRequest request, Provider provider) {
String host = request.getHeader("host");
if (host == null) {
String referrer = request.getHeader("referer");
if (referrer != null) {
URI url;
try {
url = new URI(referrer);
int port = url.getPort();
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
} catch (URISyntaxException e) {
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
}
} else {
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to.");
}
}
return "http://" + host + "/command/freebase-extension/authorize/" + provider.getHost();
}
}

47
extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java Normal file
View File

@ -0,0 +1,47 @@
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class CheckAuthorizationCommand extends Command {
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
// this cookie should not be there, but this is good hygiene practice
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
if (access_credentials != null) {
String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
response.getWriter().write(user_info);
} else {
respond(response, "401 Unauthorized", "You don't have the right credentials");
}
} catch (Exception e) {
logger.info("error",e);
respondException(response, e);
}
}
}

32
extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java Normal file
View File

@ -0,0 +1,32 @@
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.refine.commands.Command;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class DeAuthorizeCommand extends Command {
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
Provider provider = OAuthUtilities.getProvider(request);
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
respond(response, "200 OK", "");
} catch (Exception e) {
respondException(response, e);
}
}
}

37
extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java Normal file
View File

@ -0,0 +1,37 @@
package com.google.refine.freebase.commands.auth;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.commands.Command;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class GetUserBadgesCommand extends Command {
final static Logger logger = LoggerFactory.getLogger("get-version_command");
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
try {
Provider provider = OAuthUtilities.getProvider(request);
String user_id = request.getParameter("user_id");
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
response.getWriter().write(user_badges);
} catch (Exception e) {
logger.info("error",e);
respondException(response, e);
}
}
}

431
extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java Normal file
View File

@ -0,0 +1,431 @@
package com.google.refine.freebase.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Serializable;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.history.Change;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.freebase.model.recon.DataExtensionReconConfig;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class DataExtensionChange implements Change {
final protected String _baseColumnName;
final protected int _columnInsertIndex;
final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes;
final protected List<Integer> _rowIndices;
final protected List<DataExtension> _dataExtensions;
protected long _historyEntryID;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
public DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
long historyEntryID
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_historyEntryID = historyEntryID;
}
protected DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
_firstNewCellIndex = firstNewCellIndex;
_oldRows = oldRows;
_newRows = newRows;
}
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_oldRows = new ArrayList<Row>(project.rows);
_newRows = new ArrayList<Row>(project.rows.size());
int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
int index = 0;
int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) {
if (r2 < project.rows.size()) {
Row oldRow2 = project.rows.get(r2);
if (oldRow2.isCellBlank(cellIndex) &&
oldRow2.isCellBlank(keyCellIndex)) {
Row newRow = oldRow2.dup();
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
r2++;
continue;
}
}
Row newRow = new Row(cellIndex + _columnNames.size());
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
}
}
project.rows.clear();
project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
try {
project.columnModel.addColumn(_columnInsertIndex + i, column, true);
// the column might have been renamed to avoid collision
_columnNames.set(i, column.getName());
} catch (ModelException e) {
// won't get here since we set the avoid collision flag
}
}
project.update();
}
}
protected void extendRow(
Row row,
DataExtension dataExtension,
int extensionRowIndex,
Map<String, Recon> reconMap
) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.id)) {
recon = reconMap.get(rc.id);
} else {
recon = Recon.makeFreebaseRecon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "mql";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.id, recon);
}
cell = new Cell(rc.name, recon);
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
public void revert(Project project) {
synchronized (project) {
project.rows.clear();
project.rows.addAll(_oldRows);
for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnInsertIndex);
}
project.update();
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) {
writer.write(name); writer.write('\n');
}
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (FreebaseType type : _columnTypes) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
type.write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
writer.write('\n');
}
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
if (dataExtension == null) {
writer.write('\n');
continue;
}
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
for (Row row : _oldRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null;
int columnInsertIndex = -1;
List<String> columnNames = null;
List<FreebaseType> columnTypes = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Row> oldRows = null;
List<Row> newRows = null;
int firstNewCellIndex = -1;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) {
baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value);
rowIndices = new ArrayList<Integer>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
rowIndices.add(Integer.parseInt(line));
}
}
} else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value);
columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
columnNames.add(line);
}
}
} else if ("columnTypeCount".equals(field)) {
int count = Integer.parseInt(value);
columnTypes = new ArrayList<FreebaseType>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
}
} else if ("dataExtensionCount".equals(field)) {
int count = Integer.parseInt(value);
dataExtensions = new ArrayList<DataExtension>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line == null) continue;
if (line.length() == 0) {
dataExtensions.add(null);
continue;
}
int rowCount = Integer.parseInt(line);
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
}
} else if ("oldRowCount".equals(field)) {
int count = Integer.parseInt(value);
oldRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line, pool));
}
}
} else if ("newRowCount".equals(field)) {
int count = Integer.parseInt(value);
newRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line, pool));
}
}
}
}
DataExtensionChange change = new DataExtensionChange(
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows
);
return change;
}
}

66
extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java Normal file
View File

@ -0,0 +1,66 @@
package com.google.refine.freebase.model.recon;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.freebase.model.recon.StrictReconConfig;
public class DataExtensionReconConfig extends StrictReconConfig {
final public FreebaseType type;
private final static String WARN = "Not implemented";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject type = obj.getJSONObject("type");
return new DataExtensionReconConfig(
new FreebaseType(
type.getString("id"),
type.getString("name")
)
);
}
public DataExtensionReconConfig(FreebaseType type) {
this.type = type;
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
throw new RuntimeException(WARN);
}
@Override
public int getBatchSize() {
throw new RuntimeException(WARN);
}
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("mode"); writer.value("extend");
writer.key("type"); type.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
throw new RuntimeException(WARN);
}
@Override
public String getBriefDescription(Project project, String columnName) {
throw new RuntimeException(WARN);
}
}

176
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java Normal file
View File

@ -0,0 +1,176 @@
package com.google.refine.freebase.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class GuidBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new GuidBasedReconConfig();
}
public GuidBasedReconConfig() {
}
static protected class GuidBasedReconJob extends ReconJob {
String guid;
public int getKey() {
return guid.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
GuidBasedReconJob job = new GuidBasedReconJob();
String s = cell.value.toString();
if (s.startsWith("/guid/")) {
s = "#" + s.substring(6);
} else if (!s.startsWith("#")) {
s = "#" + s;
}
job.guid = s;
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("guid|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((GuidBasedReconJob) job).guid);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String guid = result.getString("guid");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String guid = ((GuidBasedReconJob) jobs.get(i)).guid;
Recon recon = guidToRecon.get(guid);
recons.add(recon);
}
return recons;
}
}

181
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java Normal file
View File

@ -0,0 +1,181 @@
package com.google.refine.freebase.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class IdBasedReconConfig extends StrictReconConfig {
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
return new IdBasedReconConfig();
}
public IdBasedReconConfig() {
}
static protected class IdBasedReconJob extends ReconJob {
String id;
public int getKey() {
return id.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
IdBasedReconJob job = new IdBasedReconJob();
String s = cell.value.toString();
if (!s.startsWith("/")) {
if (s.startsWith("92")) {
s = "/guid/" + s;
} else if (!s.contains("/")){
s = "/en/" + s;
} else {
s = "/" + s;
}
}
job.id = s;
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " as Freebase IDs";
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("id");
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> idToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("id|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((IdBasedReconJob) job).id);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String id = result.getString("id");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String id = ((IdBasedReconJob) jobs.get(i)).id;
Recon recon = idToRecon.get(id);
recons.add(recon);
}
return recons;
}
}

195
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java Normal file
View File

@ -0,0 +1,195 @@
package com.google.refine.freebase.model.recon;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.util.ParsingUtilities;
public class KeyBasedReconConfig extends StrictReconConfig {
final public FreebaseTopic namespace;
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
JSONObject ns = obj.getJSONObject("namespace");
return new KeyBasedReconConfig(
new FreebaseTopic(
ns.getString("id"),
ns.getString("name")
)
);
}
public KeyBasedReconConfig(FreebaseTopic namespace) {
this.namespace = namespace;
}
static protected class KeyBasedReconJob extends ReconJob {
String key;
public int getKey() {
return key.hashCode();
}
}
@Override
public ReconJob createJob(Project project, int rowIndex, Row row,
String columnName, Cell cell) {
KeyBasedReconJob job = new KeyBasedReconJob();
job.key = cell.value.toString().replace(' ', '_');
return job;
}
@Override
public int getBatchSize() {
return 10;
}
@Override
public String getBriefDescription(Project project, String columnName) {
return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("strict");
writer.key("match"); writer.value("key");
writer.key("namespace"); namespace.write(writer, options);
writer.endObject();
}
@Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size());
Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
try {
String query = null;
{
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("name"); jsonWriter.value(null);
jsonWriter.key("guid"); jsonWriter.value(null);
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
jsonWriter.key("key");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("namespace");
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(namespace.id);
jsonWriter.endObject();
jsonWriter.key("value"); jsonWriter.value(null);
jsonWriter.key("value|=");
jsonWriter.array();
for (ReconJob job : jobs) {
jsonWriter.value(((KeyBasedReconJob) job).key);
}
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
}
for (int i = 0; i < jobs.size(); i++) {
String key = ((KeyBasedReconJob) jobs.get(i)).key;
Recon recon = keyToRecon.get(key);
recons.add(recon);
}
return recons;
}
}

21
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java Normal file
View File

@ -0,0 +1,21 @@
package com.google.refine.freebase.model.recon;
import org.json.JSONObject;
import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match");
if ("key".equals(match)) {
return KeyBasedReconConfig.reconstruct(obj);
} else if ("id".equals(match)) {
return IdBasedReconConfig.reconstruct(obj);
} else if ("guid".equals(match)) {
return GuidBasedReconConfig.reconstruct(obj);
}
return null;
}
}

37
extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java Normal file
View File

@ -0,0 +1,37 @@
package com.google.refine.freebase.oauth;
import oauth.signpost.OAuthConsumer;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
import com.google.refine.freebase.util.FreebaseUtils;
public class FreebaseProvider extends Provider {
static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"};
static public void register() {
OAuthUtilities.registerOAuthProvider(new FreebaseProvider(FreebaseUtils.FREEBASE_HOST), FREEBASE_OAUTH_INFO);
}
public FreebaseProvider(String host) {
super(host);
}
public String getRequestTokenServiceURL() {
return "https://" + host + "/api/oauth/request_token";
}
public String getAccessTokenServiceURL() {
return "https://" + host + "/api/oauth/access_token";
}
public String getUserAuthorizationURL() {
return "https://" + host + "/signin/app";
}
@Override
public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) {
return new FreebaseTimeCommonsHttpOAuthConsumer(consumerKey, consumerSecret);
}
}

69
extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java Normal file
View File

@ -0,0 +1,69 @@
package com.google.refine.freebase.oauth;
import java.io.IOException;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer {
final static Logger logger = LoggerFactory.getLogger("oauth");
private static final long serialVersionUID = -4139931605235255279L;
private static final int SOCKET_TIMEOUT = 3000;
private static final int CONNECTION_TIMEOUT = 3000;
private static final String TIMER_URL = "http://refinery.freebaseapps.com/time";
public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) {
super(consumerKey, consumerSecret);
}
/**
* It might be that the user's computer's clock is not synchronized enough with the Freebase servers
* and this might result in Freebase thinking that it was under a replay attack.
* To avoid this problem we get the timestamp directly from acre that we know is synchronized.
*
* NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same
* could be said if we used an NTP client.
*/
protected String generateTimestamp() {
long time = -1;
try {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT);
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT);
HttpClient httpClient = new DefaultHttpClient(httpParams);
HttpGet httpget = new HttpGet(TIMER_URL);
HttpResponse response = httpClient.execute(httpget);
HttpEntity entity = response.getEntity();
if (entity != null) {
time = Long.parseLong(EntityUtils.toString(entity),10);
logger.debug("Got remote timestamp {}", time);
}
} catch (IOException e) {
logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e);
}
if (time == -1) {
time = System.currentTimeMillis();
}
return Long.toString(time / 1000L);
}
}

275
extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java Normal file
View File

@ -0,0 +1,275 @@
package com.google.refine.freebase.operations;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.freebase.util.FreebaseDataExtensionJob;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellAtRow;
import com.google.refine.freebase.model.changes.DataExtensionChange;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
public class ExtendDataOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected JSONObject _extension;
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ExtendDataOperation(
engineConfig,
obj.getString("baseColumnName"),
obj.getJSONObject("extension"),
obj.getInt("columnInsertIndex")
);
}
public ExtendDataOperation(
JSONObject engineConfig,
String baseColumnName,
JSONObject extension,
int columnInsertIndex
) {
super(engineConfig);
_baseColumnName = baseColumnName;
_extension = extension;
_columnInsertIndex = columnInsertIndex;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); writer.value(getEngineConfig());
writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
writer.key("baseColumnName"); writer.value(_baseColumnName);
writer.key("extension"); writer.value(_extension);
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + _baseColumnName;
}
protected String createDescription(Column column, List<CellAtRow> cellsAtRows) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + column.getName() +
" by filling " + cellsAtRows.size();
}
public Process createProcess(Project project, Properties options) throws Exception {
return new ExtendDataProcess(
project,
getEngineConfig(),
getBriefDescription(null)
);
}
public class ExtendDataProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected JSONObject _engineConfig;
final protected long _historyEntryID;
protected int _cellIndex;
protected FreebaseDataExtensionJob _job;
public ExtendDataProcess(
Project project,
JSONObject engineConfig,
String description
) throws JSONException {
super(description);
_project = project;
_engineConfig = engineConfig;
_historyEntryID = HistoryEntry.allocateID();
_job = new FreebaseDataExtensionJob(_extension);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(hashCode());
writer.key("description"); writer.value(_description);
writer.key("immediate"); writer.value(false);
writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done"));
writer.key("progress"); writer.value(_progress);
writer.endObject();
}
protected Runnable getRunnable() {
return this;
}
protected void populateRowsWithMatches(List<Integer> rowIndices) throws Exception {
Engine engine = new Engine(_project);
engine.initializeFromJSON(_engineConfig);
Column column = _project.columnModel.getColumnByName(_baseColumnName);
if (column == null) {
throw new Exception("No column named " + _baseColumnName);
}
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
return false;
}
}.init(rowIndices));
}
protected int extendRows(
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
Map<String, ReconCandidate> reconCandidateMap
) {
Set<String> ids = new HashSet<String>();
int end;
for (end = from; end < limit && ids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
ids.add(cell.recon.match.id);
}
Map<String, DataExtension> map = null;
try {
map = _job.extend(ids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.id;
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
return end;
}
public void run() {
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try {
populateRowsWithMatches(rowIndices);
} catch (Exception e2) {
// TODO : Not sure what to do here?
e2.printStackTrace();
}
int start = 0;
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
_progress = end * 100 / rowIndices.size();
try {
Thread.sleep(200);
} catch (InterruptedException e) {
if (_canceled) {
break;
}
}
}
if (!_canceled) {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) {
columnTypes.add(info.expectedType);
}
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
ExtendDataOperation.this,
new DataExtensionChange(
_baseColumnName,
_columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
_historyEntryID)
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
}
}
}

106
extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java Normal file
View File

@ -0,0 +1,106 @@
package com.google.refine.freebase.operations;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.commands.UploadDataCommand;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassReconChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
public class ImportQADataOperation extends AbstractOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new ImportQADataOperation();
}
public ImportQADataOperation() {
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Import QA DAta";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref);
if (jobID == null) {
throw new InternalError("Project is not associated with any data loading job.");
}
Map<Long, String> reconIDToResult = new HashMap<Long, String>();
URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(30000); // 30 seconds
LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream()));
try {
String line;
while ((line = reader.readLine()) != null) {
JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line);
long reconID = Long.parseLong(obj.getString("recon_id").substring(3));
reconIDToResult.put(reconID, obj.getString("result"));
}
} finally {
reader.close();
}
Map<Long, Recon> oldRecons = new HashMap<Long, Recon>();
Map<Long, Recon> newRecons = new HashMap<Long, Recon>();
for (int r = 0; r < project.rows.size(); r++) {
Row row = project.rows.get(r);
for (int c = 0; c < row.cells.size(); c++) {
Cell cell = row.cells.get(c);
if (cell != null && cell.recon != null) {
Recon oldRecon = cell.recon;
if (reconIDToResult.containsKey(oldRecon.id)) {
Recon newRecon = oldRecon.dup();
newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id));
reconIDToResult.remove(oldRecon.id);
oldRecons.put(oldRecon.id, oldRecon);
newRecons.put(oldRecon.id, newRecon);
}
}
}
}
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(project),
this,
new MassReconChange(newRecons, oldRecons)
);
}
}

121
extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java Normal file
View File

@ -0,0 +1,121 @@
package com.google.refine.freebase.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveProtographOperation extends AbstractOperation {
final protected Protograph _protograph;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new SaveProtographOperation(
Protograph.reconstruct(obj.getJSONObject("protograph"))
);
}
public SaveProtographOperation(Protograph protograph) {
_protograph = protograph;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription());
writer.key("protograph"); _protograph.write(writer, options);
writer.endObject();
}
protected String getBriefDescription() {
return "Save schema alignment skeleton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Change change = new ProtographChange(_protograph);
return new HistoryEntry(historyEntryID, project, getBriefDescription(), SaveProtographOperation.this, change);
}
static public class ProtographChange implements Change {
final protected Protograph _newProtograph;
protected Protograph _oldProtograph;
public ProtographChange(Protograph protograph) {
_newProtograph = protograph;
}
public void apply(Project project) {
synchronized (project) {
_oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph");
project.overlayModels.put("freebaseProtograph", _newProtograph);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldProtograph == null) {
project.overlayModels.remove("freebaseProtograph");
} else {
project.overlayModels.put("freebaseProtograph", _oldProtograph);
}
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
Protograph oldProtograph = null;
Protograph newProtograph = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldProtograph".equals(field) && value.length() > 0) {
oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newProtograph".equals(field) && value.length() > 0) {
newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
ProtographChange change = new ProtographChange(newProtograph);
change._oldProtograph = oldProtograph;
return change;
}
static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
if (p != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
p.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

47
extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java Normal file
View File

@ -0,0 +1,47 @@
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
public class AnonymousNode implements Node, NodeWithLinks {
final public FreebaseType type;
final public List<Link> links = new LinkedList<Link>();
public AnonymousNode(FreebaseType type) {
this.type = type;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("anonymous");
writer.key("type"); type.write(writer, options);
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
public void addLink(Link link) {
links.add(link);
}
public Link getLink(int index) {
return links.get(index);
}
public int getLinkCount() {
return links.size();
}
}

42
extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java Normal file
View File

@ -0,0 +1,42 @@
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
public class BooleanColumnCondition implements Condition {
final public String columnName;
public BooleanColumnCondition(String columnName) {
this.columnName = columnName;
}
@Override
public boolean test(Project project, int rowIndex, Row row) {
Column column = project.columnModel.getColumnByName(columnName);
if (column != null) {
Object o = row.getCellValue(column.getCellIndex());
if (o != null) {
if (o instanceof Boolean) {
return ((Boolean) o).booleanValue();
} else {
return Boolean.parseBoolean(o.toString());
}
}
}
return false;
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("columnName"); writer.value(columnName);
writer.endObject();
}
}

35
extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java Normal file
View File

@ -0,0 +1,35 @@
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
public class CellKeyNode extends CellNode {
final public FreebaseTopic namespace;
public CellKeyNode(
FreebaseTopic namespace
) {
this.namespace = namespace;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-key");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
writer.key("namespace"); namespace.write(writer, options);
writer.endObject();
}
}

8
extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java Normal file
View File

@ -0,0 +1,8 @@
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
abstract public class CellNode implements Node {
final public List<String> columnNames = new LinkedList<String>();
}

58
extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java Normal file
View File

@ -0,0 +1,58 @@
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
public class CellTopicNode extends CellNode implements NodeWithLinks {
final public FreebaseType type;
final public List<Link> links = new LinkedList<Link>();
public CellTopicNode(
FreebaseType type
) {
this.type = type;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-topic");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
if (type != null) {
writer.key("type"); type.write(writer, options);
}
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
public void addLink(Link link) {
links.add(link);
}
public Link getLink(int index) {
return links.get(index);
}
public int getLinkCount() {
return links.size();
}
}

36
extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java Normal file
View File

@ -0,0 +1,36 @@
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
public class CellValueNode extends CellNode {
final public String valueType;
final public String lang;
public CellValueNode(
String valueType,
String lang
) {
this.valueType = valueType;
this.lang = lang;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("cell-as-value");
writer.key("columnNames");
writer.array();
for (String name : columnNames) {
writer.value(name);
}
writer.endArray();
writer.key("valueType"); writer.value(valueType);
writer.key("lang"); writer.value(lang);
writer.endObject();
}
}

9
extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java Normal file
View File

@ -0,0 +1,9 @@
package com.google.refine.freebase.protograph;
import com.google.refine.Jsonizable;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
public interface Condition extends Jsonizable {
public boolean test(Project project, int rowIndex, Row row);
}

48
extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java Normal file
View File

@ -0,0 +1,48 @@
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseTopic;
public class FreebaseTopicNode implements Node, NodeWithLinks {
final public FreebaseTopic topic;
final public List<Link> links = new LinkedList<Link>();
public FreebaseTopicNode(FreebaseTopic topic) {
this.topic = topic;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("topic");
writer.key("topic"); topic.write(writer, options);
if (links != null) {
writer.key("links"); writer.array();
for (Link link : links) {
link.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
public void addLink(Link link) {
links.add(link);
}
public Link getLink(int index) {
return links.get(index);
}
public int getLinkCount() {
return links.size();
}
}

48
extensions/freebase/src/com/google/refine/freebase/protograph/Link.java Normal file
View File

@ -0,0 +1,48 @@
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.freebase.FreebaseProperty;
public class Link implements Jsonizable {
final public FreebaseProperty property;
final public Node target;
final public Condition condition;
final public boolean load;
public Link(FreebaseProperty property, Node target, Condition condition, boolean load) {
this.property = property;
this.target = target;
this.condition = condition;
this.load = load;
}
public FreebaseProperty getProperty() {
return property;
}
public Node getTarget() {
return target;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("property"); property.write(writer, options);
if (target != null) {
writer.key("target");
target.write(writer, options);
}
if (condition != null) {
writer.key("condition");
condition.write(writer, options);
}
writer.endObject();
}
}

6
extensions/freebase/src/com/google/refine/freebase/protograph/Node.java Normal file
View File

@ -0,0 +1,6 @@
package com.google.refine.freebase.protograph;
import com.google.refine.Jsonizable;
public interface Node extends Jsonizable {
}

9
extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java Normal file
View File

@ -0,0 +1,9 @@
package com.google.refine.freebase.protograph;
public interface NodeWithLinks {
public void addLink(Link link);
public int getLinkCount();
public Link getLink(int index);
}

171
extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java Normal file
View File

@ -0,0 +1,171 @@
package com.google.refine.freebase.protograph;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
public class Protograph implements OverlayModel {
final protected List<Node> _rootNodes = new LinkedList<Node>();
public int getRootNodeCount() {
return _rootNodes.size();
}
public Node getRootNode(int index) {
return _rootNodes.get(index);
}
@Override
public void onBeforeSave(Project project) {
}
@Override
public void onAfterSave(Project project) {
}
@Override
public void dispose(Project project) {
}
static public Protograph reconstruct(JSONObject o) throws JSONException {
Protograph g = new Protograph();
JSONArray rootNodes = o.getJSONArray("rootNodes");
int count = rootNodes.length();
for (int i = 0; i < count; i++) {
JSONObject o2 = rootNodes.getJSONObject(i);
Node node = reconstructNode(o2);
if (node != null) {
g._rootNodes.add(node);
}
}
return g;
}
static protected Node reconstructNode(JSONObject o) throws JSONException {
Node node = null;
String nodeType = o.getString("nodeType");
if (nodeType.startsWith("cell-as-")) {
if ("cell-as-topic".equals(nodeType)) {
if (o.has("type")) {
node = new CellTopicNode(
reconstructType(o.getJSONObject("type"))
);
}
} else if ("cell-as-value".equals(nodeType)) {
node = new CellValueNode(
o.getString("valueType"),
o.getString("lang")
);
} else if ("cell-as-key".equals(nodeType)) {
node = new CellKeyNode(
reconstructTopic(o.getJSONObject("namespace"))
);
}
if (o.has("columnName") && !o.isNull("columnName")) {
((CellNode) node).columnNames.add(o.getString("columnName"));
}
if (o.has("columnNames") && !o.isNull("columnNames")) {
JSONArray columnNames = o.getJSONArray("columnNames");
int count = columnNames.length();
for (int c = 0; c < count; c++) {
((CellNode) node).columnNames.add(columnNames.getString(c));
}
}
} else if ("topic".equals(nodeType)) {
node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
} else if ("value".equals(nodeType)) {
node = new ValueNode(
o.get("value"),
o.getString("valueType"),
o.getString("lang")
);
} else if ("anonymous".equals(nodeType)) {
node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
}
if (node != null && node instanceof NodeWithLinks && o.has("links")) {
NodeWithLinks node2 = (NodeWithLinks) node;
JSONArray links = o.getJSONArray("links");
int linkCount = links.length();
for (int j = 0; j < linkCount; j++) {
JSONObject oLink = links.getJSONObject(j);
Condition condition = null;
if (oLink.has("condition") && !oLink.isNull("condition")) {
JSONObject oCondition = oLink.getJSONObject("condition");
if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
condition = new BooleanColumnCondition(oCondition.getString("columnName"));
}
}
node2.addLink(new Link(
reconstructProperty(oLink.getJSONObject("property")),
oLink.has("target") && !oLink.isNull("target") ?
reconstructNode(oLink.getJSONObject("target")) : null,
condition,
oLink.has("load") && !oLink.isNull("load") ?
oLink.getBoolean("load") : true
));
}
}
return node;
}
static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
return new FreebaseProperty(
o.getString("id"),
o.getString("name")
);
}
static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
return new FreebaseType(
o.getString("id"),
o.getString("name")
);
}
static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
return new FreebaseTopic(
o.getString("id"),
o.getString("name")
);
}
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("rootNodes"); writer.array();
for (Node node : _rootNodes) {
node.write(writer, options);
}
writer.endArray();
writer.endObject();
}
static public Protograph load(Project project, JSONObject obj) throws Exception {
return reconstruct(obj);
}
}

29
extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java Normal file
View File

@ -0,0 +1,29 @@
package com.google.refine.freebase.protograph;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
public class ValueNode implements Node {
final public Object value;
final public String valueType;
final public String lang;
public ValueNode(Object value, String valueType, String lang) {
this.value = value;
this.valueType = valueType;
this.lang = lang;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nodeType"); writer.value("value");
writer.key("value"); writer.value(value);
writer.key("valueType"); writer.value(valueType);
writer.key("lang"); writer.value(lang);
writer.endObject();
}
}

346
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java Normal file
View File

@ -0,0 +1,346 @@
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellKeyNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.CellValueNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Recon;
import com.google.refine.util.JSONUtilities;
public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
protected Writer writer;
protected List<JSONObject> rootObjects = new LinkedList<JSONObject>();
private static final String TYPE = "type";
private static final String ID = "id";
private static final String NAME = "name";
private static final String CREATE = "create";
private static final String VALUE = "value";
private static final String CONNECT = "connect";
private static final String LANG = "lang";
public MqlwriteLikeTransposedNodeFactory(Writer writer) {
this.writer = writer;
}
protected JSONArray getJSON() {
return new JSONArray(rootObjects);
}
@Override
public void flush() throws IOException {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.array();
for (JSONObject obj : rootObjects) {
jsonWriter.value(obj);
}
jsonWriter.endArray();
} catch (JSONException e) {
e.printStackTrace();
}
writer.flush();
}
abstract protected class JsonTransposedNode implements TransposedNode {
abstract public Object getJSON();
}
abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
abstract public JSONObject getJSONObject();
protected JSONObject obj;
public Object getJSON() {
return getJSONObject();
}
}
protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
JsonObjectTransposedNode parent;
FreebaseProperty property;
AnonymousNode node;
protected AnonymousTransposedNode(
JsonObjectTransposedNode parent,
FreebaseProperty property,
AnonymousNode node
) {
this.parent = parent;
this.property = property;
this.node = node;
}
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(TYPE, this.node.type.id);
obj.put(ID, (String) null);
obj.put(CREATE, "unconditional");
} catch (JSONException e) {
e.printStackTrace();
}
linkTransposedNodeJSON(obj, parent, property);
}
return obj;
}
}
protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
protected CellTopicNode node;
protected Cell cell;
public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
@Override
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
if (cell.recon != null &&
cell.recon.judgment == Recon.Judgment.Matched &&
cell.recon.match != null) {
obj.put(ID, cell.recon.match.id);
} else {
obj.put(ID, (String) null);
obj.put(NAME, cell.value.toString());
obj.put(TYPE, node.type.id);
obj.put(CREATE, "unless_exists");
}
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class CellValueTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected CellValueNode node;
protected Cell cell;
public CellValueTransposedNode(CellValueNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
JSONUtilities.putField(obj, VALUE, cell.value);
obj.put(TYPE, node.valueType);
if ("/type/text".equals(node.valueType)) {
obj.put(LANG, node.lang);
}
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class CellKeyTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected CellKeyNode node;
protected Cell cell;
public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
this.node = node;
this.cell = cell;
}
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(VALUE, cell.value.toString());
JSONObject nsObj = new JSONObject();
nsObj.put(ID, node.namespace.id);
obj.put("namespace", nsObj);
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class TopicTransposedNode extends JsonObjectTransposedNode {
protected FreebaseTopicNode node;
public TopicTransposedNode(FreebaseTopicNode node) {
this.node = node;
}
@Override
public JSONObject getJSONObject() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(ID, node.topic.id);
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
protected class ValueTransposedNode extends JsonTransposedNode {
protected JSONObject obj;
protected ValueNode node;
public ValueTransposedNode(ValueNode node) {
this.node = node;
}
public Object getJSON() {
if (obj == null) {
obj = new JSONObject();
try {
obj.put(VALUE, node.value);
obj.put(TYPE, node.valueType);
if ("/type/text".equals(node.valueType)) {
obj.put(LANG, node.lang);
}
obj.put(CONNECT, "insert");
} catch (JSONException e) {
e.printStackTrace();
}
}
return obj;
}
}
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex) {
return new AnonymousTransposedNode(
parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
link != null ? link.property : null,
node
);
}
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell) {
JsonTransposedNode tnode = null;
if (node instanceof CellTopicNode) {
tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
} else if (node instanceof CellValueNode) {
tnode = new CellValueTransposedNode((CellValueNode) node, cell);
} else if (node instanceof CellKeyNode) {
tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
}
if (tnode != null) {
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
}
return tnode;
}
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node, int rowIndex) {
JsonTransposedNode tnode = new TopicTransposedNode(node);
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
return tnode;
}
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node, int rowIndex) {
JsonTransposedNode tnode = new ValueTransposedNode(node);
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
return tnode;
}
protected void processTransposedNode(
JsonTransposedNode tnode,
TransposedNode parentNode,
FreebaseProperty property
) {
if (!(tnode instanceof AnonymousTransposedNode)) {
linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
}
}
protected void linkTransposedNodeJSON(
Object obj,
TransposedNode parentNode,
FreebaseProperty property
) {
if (parentNode == null) {
if (obj instanceof JSONObject) {
rootObjects.add((JSONObject) obj);
}
} else if (parentNode instanceof JsonTransposedNode) {
JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
try {
JSONArray a = null;
if (parentObj.has(property.id)) {
a = parentObj.getJSONArray(property.id);
} else {
a = new JSONArray();
parentObj.put(property.id, a);
}
a.put(a.length(), obj);
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}

4
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java Normal file
View File

@ -0,0 +1,4 @@
package com.google.refine.freebase.protograph.transpose;
public interface TransposedNode {
}

43
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java Normal file
View File

@ -0,0 +1,43 @@
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
public interface TransposedNodeFactory {
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex
);
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell
);
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node,
int rowIndex
);
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node,
int rowIndex
);
public void flush() throws IOException;
}

222
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java Normal file
View File

@ -0,0 +1,222 @@
package com.google.refine.freebase.protograph.transpose;
import java.util.LinkedList;
import java.util.List;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.Node;
import com.google.refine.freebase.protograph.NodeWithLinks;
import com.google.refine.freebase.protograph.Protograph;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.Recon.Judgment;
public class Transposer {
static public void transpose(
Project project,
FilteredRows filteredRows,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory
) {
transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
}
static public void transpose(
Project project,
FilteredRows filteredRows,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory,
int limit
) {
Context rootContext = new Context(rootNode, null, null, limit);
filteredRows.accept(project, new RowVisitor() {
Context rootContext;
Protograph protograph;
Node rootNode;
TransposedNodeFactory nodeFactory;
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
}
if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
return true;
}
return false;
}
@Override
public void start(Project project) {
// TODO Auto-generated method stub
}
@Override
public void end(Project project) {
// TODO Auto-generated method stub
}
public RowVisitor init(
Context rootContext,
Protograph protograph,
Node rootNode,
TransposedNodeFactory nodeFactory
) {
this.rootContext = rootContext;
this.protograph = protograph;
this.rootNode = rootNode;
this.nodeFactory = nodeFactory;
return this;
}
}.init(rootContext, protograph, rootNode, nodeFactory));
}
static protected void descend(
Project project,
Protograph protograph,
TransposedNodeFactory nodeFactory,
int rowIndex,
Row row,
Node node,
Context context
) {
List<TransposedNode> tnodes = new LinkedList<TransposedNode>();
TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode;
Link link = context.parent == null ? null : context.link;
if (node instanceof CellNode) {
CellNode node2 = (CellNode) node;
for (String columnName : node2.columnNames) {
Column column = project.columnModel.getColumnByName(columnName);
if (column != null) {
int cellIndex = column.getCellIndex();
Cell cell = row.getCell(cellIndex);
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
if (node2 instanceof CellTopicNode &&
(cell.recon == null || cell.recon.judgment == Judgment.None)) {
return;
}
context.count++;
if (context.limit > 0 && context.count > context.limit) {
return;
}
tnodes.add(nodeFactory.transposeCellNode(
parentNode,
link,
node2,
rowIndex,
cellIndex,
cell
));
}
}
}
} else {
if (node instanceof AnonymousNode) {
tnodes.add(nodeFactory.transposeAnonymousNode(
parentNode,
link,
(AnonymousNode) node,
rowIndex
));
} else if (node instanceof FreebaseTopicNode) {
tnodes.add(nodeFactory.transposeTopicNode(
parentNode,
link,
(FreebaseTopicNode) node,
rowIndex
));
} else if (node instanceof ValueNode) {
tnodes.add(nodeFactory.transposeValueNode(
parentNode,
link,
(ValueNode) node,
rowIndex
));
}
}
if (node instanceof NodeWithLinks) {
NodeWithLinks node2 = (NodeWithLinks) node;
int linkCount = node2.getLinkCount();
for (int i = 0; i < linkCount; i++) {
Link link2 = node2.getLink(i);
if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
for (TransposedNode tnode : tnodes) {
context.transposedNode = tnode;
context.nullifySubContextNodes();
descend(
project,
protograph,
nodeFactory,
rowIndex,
row,
link2.getTarget(),
context.subContexts.get(i)
);
}
}
}
}
}
static class Context {
TransposedNode transposedNode;
List<Context> subContexts;
Context parent;
Link link;
int count;
int limit;
Context(Node node, Context parent, Link link, int limit) {
this.parent = parent;
this.link = link;
this.limit = limit;
if (node instanceof NodeWithLinks) {
NodeWithLinks node2 = (NodeWithLinks) node;
int subContextCount = node2.getLinkCount();
subContexts = new LinkedList<Context>();
for (int i = 0; i < subContextCount; i++) {
Link link2 = node2.getLink(i);
subContexts.add(
new Context(link2.getTarget(), this, link2, -1));
}
}
}
public void nullifySubContextNodes() {
if (subContexts != null) {
for (Context context : subContexts) {
context.transposedNode = null;
context.nullifySubContextNodes();
}
}
}
}
}

725
extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java Normal file
View File

@ -0,0 +1,725 @@
package com.google.refine.freebase.protograph.transpose;
import java.io.IOException;
import java.io.Writer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.freebase.FreebaseProperty;
import com.google.refine.freebase.FreebaseTopic;
import com.google.refine.freebase.protograph.AnonymousNode;
import com.google.refine.freebase.protograph.CellKeyNode;
import com.google.refine.freebase.protograph.CellNode;
import com.google.refine.freebase.protograph.CellTopicNode;
import com.google.refine.freebase.protograph.CellValueNode;
import com.google.refine.freebase.protograph.FreebaseTopicNode;
import com.google.refine.freebase.protograph.Link;
import com.google.refine.freebase.protograph.ValueNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.StandardReconConfig;
public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
protected Project project;
protected boolean start = true;
protected Writer writer;
protected WritingTransposedNode lastRootNode;
protected Map<String, Long> varPool = new HashMap<String, Long>();
protected Map<Long, String> newTopicVars = new HashMap<Long, String>();
protected Set<Long> serializedRecons = new HashSet<Long>();
protected long contextID = 0;
protected int contextRowIndex;
protected int contextRefCount = 0;
protected JSONObject contextTreeRoot;
public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
this.project = project;
this.writer = writer;
}
@Override
public void flush() throws IOException {
if (lastRootNode != null) {
lastRootNode.write(null, null, project, -1, -1, null);
lastRootNode = null;
writeContextTreeNode();
}
}
protected void writeLine(String line) {
try {
if (start) {
start = false;
} else {
writer.write('\n');
}
writer.write(line);
} catch (IOException e) {
// ignore
}
}
protected void writeRecon(
StringBuffer sb,
Project project,
int rowIndex,
int cellIndex,
Cell cell
) {
Recon recon = cell.recon;
sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
contextRefCount++;
if (!serializedRecons.contains(recon.id)) {
serializedRecons.add(recon.id);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
// qa:sample_group
{
StringBuffer sb2 = new StringBuffer();
sb2.append("{ \"s\" : \"rec");
sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
sb2.append(JSONObject.quote(column.getName()));
sb2.append(", \"ignore\" : true }");
writeLine(sb2.toString());
}
// qa:recon_data
{
StringBuffer sb2 = new StringBuffer();
String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
sb2.append("{ \"s\" : \"rec");
sb2.append(Long.toString(recon.id));
sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
if (recon.judgment == Judgment.Matched) {
sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
}
ReconConfig reconConfig = column.getReconConfig();
if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
}
sb2.append(" } }");
writeLine(sb2.toString());
}
}
}
protected void writeLine(
String subject, String predicate, Object object,
Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
int objectRowIndex, int objectCellIndex, Cell objectCell,
boolean ignore
) {
if (subject != null && object != null) {
String s = object instanceof String ?
JSONObject.quote((String) object) : object.toString();
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : "); sb.append(s);
if (subjectCell != null || objectCell != null) {
sb.append(", \"meta\" : { ");
sb.append("\"recon\" : { ");
if (subjectCell != null) {
sb.append("\"s\" : ");
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
}
if (objectCell != null) {
if (subjectCell != null) {
sb.append(", ");
}
sb.append("\"o\" : ");
writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
}
sb.append(" }");
sb.append(" }");
}
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }");
writeLine(sb.toString());
}
}
protected void writeLine(
String subject, String predicate, Object object, String lang,
Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
boolean ignore
) {
if (subject != null && object != null) {
String s = object instanceof String ?
JSONObject.quote((String) object) : object.toString();
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : "); sb.append(s);
sb.append(", \"lang\" : "); sb.append(lang);
if (subjectCell != null) {
sb.append(", \"meta\" : { ");
sb.append("\"recon\" : { ");
sb.append("\"s\" : ");
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
sb.append(" }");
sb.append(" }");
}
if (ignore) {
sb.append(", \"ignore\" : true");
}
sb.append(" }");
writeLine(sb.toString());
}
}
abstract protected class WritingTransposedNode implements TransposedNode {
JSONObject jsonContextNode;
boolean load;
public Object write(
String subject, String predicate, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
return internalWrite(
subject, predicate, project,
subjectRowIndex, subjectCellIndex, subjectCell);
}
abstract public Object internalWrite(
String subject, String predicate, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
}
abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
public List<Link> links = new LinkedList<Link>();
public List<Integer> rowIndices = new LinkedList<Integer>();
public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
protected void writeChildren(
String subject, Project project,
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
for (int i = 0; i < children.size(); i++) {
WritingTransposedNode child = children.get(i);
Link link = links.get(i);
String predicate = link.property.id;
child.write(subject, predicate, project,
subjectRowIndex, subjectCellIndex, subjectCell);
}
}
}
protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
//protected AnonymousTransposedNode(AnonymousNode node) { }
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
if (children.size() == 0 || subject == null) {
return null;
}
StringBuffer sb = new StringBuffer();
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
sb.append(", \"o\" : { ");
StringBuffer sbRecon = new StringBuffer();
boolean first = true;
boolean firstRecon = true;
if (subjectCell.recon != null) {
sbRecon.append("\"s\" : ");
writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
firstRecon = false;
}
for (int i = 0; i < children.size(); i++) {
WritingTransposedNode child = children.get(i);
Link link = links.get(i);
FreebaseProperty property = link.property;
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
if (c != null) {
if (first) {
first = false;
} else {
sb.append(", ");
}
sb.append("\"" + property.id + "\": ");
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
}
if (child instanceof CellTopicTransposedNode) {
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
Recon recon = child2.cell.recon;
if (recon != null &&
(recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
if (firstRecon) {
firstRecon = false;
} else {
sbRecon.append(", ");
}
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
writeRecon(sbRecon, project,
rowIndices.get(i), child2.cellIndex, child2.cell);
}
}
}
sb.append(" }, \"meta\" : { \"recon\" : { ");
sb.append(sbRecon.toString());
sb.append(" } } }");
writeLine(sb.toString());
return null;
}
}
protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
protected CellTopicNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
String id = null;
if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
int objectRowIndex = rowIndex;
int objectCellIndex = cellIndex;
Cell objectCell = cell;
if (cell.recon.judgment == Recon.Judgment.Matched) {
id = cell.recon.match.id;
} else if (cell.recon.judgment == Judgment.New) {
if (newTopicVars.containsKey(cell.recon.id)) {
id = newTopicVars.get(cell.recon.id);
} else {
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
String columnName = column.getName();
long var = 0;
if (varPool.containsKey(columnName)) {
var = varPool.get(columnName);
}
varPool.put(columnName, var + 1);
id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
String typeID = node.type.id;
ReconConfig reconConfig = column.getReconConfig();
if (reconConfig instanceof StandardReconConfig) {
typeID = ((StandardReconConfig) reconConfig).typeID;
}
writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
if (cell.recon != null) {
newTopicVars.put(cell.recon.id, id);
}
}
} else {
return null;
}
if (subject != null) {
writeLine(subject, predicate, id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
objectRowIndex, objectCellIndex, objectCell, !load);
}
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
}
return id;
}
}
protected class CellValueTransposedNode extends WritingTransposedNode {
protected JSONObject obj;
protected CellValueNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
if (subject != null) {
if ("/type/text".equals(node.lang)) {
writeLine(subject, predicate, cell.value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell, !load);
} else {
writeLine(subject, predicate, cell.value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
}
}
return cell.value;
}
}
protected class CellKeyTransposedNode extends WritingTransposedNode {
protected CellKeyNode node;
protected int rowIndex;
protected int cellIndex;
protected Cell cell;
public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
this.node = node;
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
this.cell = cell;
}
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
return null;
}
}
protected class TopicTransposedNode extends TransposedNodeWithChildren {
protected FreebaseTopicNode node;
public TopicTransposedNode(FreebaseTopicNode node) {
this.node = node;
}
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
writeLine(subject, predicate, node.topic.id, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
writeChildren(node.topic.id, project, -1, -1, null);
return node.topic.id;
}
}
protected class ValueTransposedNode extends WritingTransposedNode {
protected ValueNode node;
public ValueTransposedNode(ValueNode node) {
this.node = node;
}
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
if ("/type/text".equals(node.lang)) {
writeLine(subject, predicate, node.value, node.lang, project,
subjectRowIndex, subjectCellIndex, subjectCell, !load);
} else {
writeLine(subject, predicate, node.value, project,
subjectRowIndex, subjectCellIndex, subjectCell,
-1, -1, null, !load);
}
return node.value;
}
}
public TransposedNode transposeAnonymousNode(
TransposedNode parentNode,
Link link,
AnonymousNode node, int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new AnonymousTransposedNode();
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
null
);
return tnode;
}
public TransposedNode transposeCellNode(
TransposedNode parentNode,
Link link,
CellNode node,
int rowIndex,
int cellIndex,
Cell cell) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = null;
if (node instanceof CellTopicNode) {
if (cell.recon != null &&
(cell.recon.judgment == Judgment.Matched ||
cell.recon.judgment == Judgment.New)) {
tnode = new CellTopicTransposedNode(
(CellTopicNode) node, rowIndex, cellIndex, cell);
}
} else if (node instanceof CellValueNode) {
tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
} else if (node instanceof CellKeyNode) {
tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
}
if (tnode != null) {
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
cell,
rowIndex
);
}
return tnode;
}
public TransposedNode transposeTopicNode(
TransposedNode parentNode,
Link link,
FreebaseTopicNode node,
int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new TopicTransposedNode(node);
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
node.topic
);
return tnode;
}
public TransposedNode transposeValueNode(
TransposedNode parentNode,
Link link,
ValueNode node,
int rowIndex) {
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
WritingTransposedNode tnode = new ValueTransposedNode(node);
tnode.load =
(parentNode2 == null || parentNode2.load) &&
(link == null || link.load);
processTransposedNode(tnode, parentNode, link, rowIndex);
tnode.jsonContextNode = addJsonContext(
parentNode2 != null ? parentNode2.jsonContextNode : null,
link != null ? link.property.id : null,
node.value
);
return tnode;
}
protected void processTransposedNode(
WritingTransposedNode tnode,
TransposedNode parentNode,
Link link,
int rowIndex
) {
if (parentNode != null) {
if (parentNode instanceof TransposedNodeWithChildren) {
TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
parentNode2.rowIndices.add(rowIndex);
parentNode2.children.add(tnode);
parentNode2.links.add(link);
}
} else {
addRootNode(tnode, rowIndex);
}
}
protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
JSONObject o = new JSONObject();
try {
if (value instanceof FreebaseTopic) {
FreebaseTopic topic = (FreebaseTopic) value;
o.put("id", topic.id);
o.put("name", topic.name);
} else {
o.put("v", value);
}
} catch (JSONException e) {
// ignore
}
connectJsonContext(parent, o, key);
return o;
}
protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
JSONObject o = new JSONObject();
connectJsonContext(parent, o, key);
try {
if (cell != null) {
o.put("v", cell.value);
if (cell.recon != null) {
o.put("recon", "rec" + cell.recon.id);
if (cell.recon.judgment == Judgment.Matched) {
o.put("id", cell.recon.match.id);
o.put("name", cell.recon.match.name);
}
// qa:display_context
{
StringBuffer sb2 = new StringBuffer();
sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
sb2.append(Long.toString(cell.recon.id));
sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
sb2.append(Long.toString(contextID));
sb2.append("\", \"meta\" : { \"row\" : ");
sb2.append(Integer.toString(rowIndex));
sb2.append(" } }");
writeLine(sb2.toString());
}
}
}
} catch (JSONException e) {
// ignore
}
return o;
}
protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
try {
if (parent == null) {
contextTreeRoot = o;
} else {
JSONArray a = null;
if (parent.has(key)) {
a = parent.getJSONArray(key);
} else {
a = new JSONArray();
parent.put(key, a);
}
a.put(o);
}
} catch (JSONException e) {
// ignore
}
}
protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
if (lastRootNode != null) {
lastRootNode.write(null, null, project, -1, -1, null);
writeContextTreeNode();
}
lastRootNode = tnode;
contextTreeRoot = null;
contextRowIndex = rowIndex;
contextRefCount = 0;
contextID++;
}
protected void writeContextTreeNode() {
if (contextTreeRoot != null && contextRefCount > 0) {
StringBuffer sb = new StringBuffer();
sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
sb.append(Long.toString(contextID));
sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
sb.append(Integer.toString(contextRowIndex));
sb.append(", \"data\" : ");
sb.append(contextTreeRoot.toString());
sb.append(" } }");
writeLine(sb.toString());
}
}
}

417
extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java Normal file
View File

@ -0,0 +1,417 @@
/**
*
*/
package com.google.refine.freebase.util;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.freebase.FreebaseType;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.model.ReconCandidate;
public class FreebaseDataExtensionJob {
static public class DataExtension {
final public Object[][] data;
public DataExtension(Object[][] data) {
this.data = data;
}
}
static public class ColumnInfo {
final public List<String> names;
final public List<String> path;
final public FreebaseType expectedType;
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names;
this.path = path;
this.expectedType = expectedType;
}
}
final public JSONObject extension;
final public int columnCount;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
this.extension = obj;
this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ?
countColumns(obj.getJSONArray("properties"), columns, new ArrayList<String>(), new ArrayList<String>()) : 0;
}
public Map<String, FreebaseDataExtensionJob.DataExtension> extend(
Set<String> ids,
Map<String, ReconCandidate> reconCandidateMap
) throws Exception {
StringWriter writer = new StringWriter();
formulateQuery(ids, extension, writer);
String query = writer.toString();
InputStream is = doMqlRead(query);
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
if (o.has("result")) {
JSONArray a = o.getJSONArray("result");
int l = a.length();
for (int i = 0; i < l; i++) {
JSONObject o2 = a.getJSONObject(i);
String id = o2.getString("id");
FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
if (ext != null) {
map.put(id, ext);
}
}
}
return map;
} finally {
is.close();
}
}
protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
List<Object[]> rows = new ArrayList<Object[]>();
collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
Object[][] data = new Object[rows.size()][columnCount];
rows.toArray(data);
return new DataExtension(data);
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
Object value,
Map<String, ReconCandidate> reconCandidateMap
) {
while (row >= rows.size()) {
rows.add(new Object[columnCount]);
}
rows.get(row)[col] = value;
}
protected void storeCell(
List<Object[]> rows,
int row,
int col,
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String id = obj.getString("id");
ReconCandidate rc;
if (reconCandidateMap.containsKey(id)) {
rc = reconCandidateMap.get(id);
} else {
rc = new ReconCandidate(
obj.getString("id"),
obj.getString("name"),
JSONUtilities.getStringArray(obj, "type"),
100
);
reconCandidateMap.put(id, rc);
}
storeCell(rows, row, col, rc, reconCandidateMap);
}
protected int[] collectResult(
List<Object[]> rows,
JSONObject extNode,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String propertyID = extNode.getString("id");
String expectedTypeID = extNode.getJSONObject("expected").getString("id");
JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ?
resultNode.getJSONArray(propertyID) : null;
if (expectedTypeID.startsWith("/type/")) {
if (a != null) {
int l = a.length();
for (int r = 0; r < l; r++) {
Object o = a.isNull(r) ? null : a.get(r);
if (o instanceof Serializable) {
storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap);
}
}
}
// note that we still take up a column even if we don't have any data
return new int[] { startRowIndex, startColumnIndex + 1 };
} else {
boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
if (a != null && a.length() > 0) {
int maxColIndex = startColumnIndex;
int l = a.length();
for (int r = 0; r < l; r++) {
Object v = a.isNull(r) ? null : a.get(r);
JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null;
int startColumnIndex2 = startColumnIndex;
int startRowIndex2 = startRowIndex;
if (isOwnColumn) {
if (o != null) {
storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap);
} else {
storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap);
}
}
if (hasSubProperties && o != null) {
int[] rowcol = collectResult(
rows,
extNode.getJSONArray("properties"),
o,
startRowIndex,
startColumnIndex2,
reconCandidateMap
);
startRowIndex2 = rowcol[0];
startColumnIndex2 = rowcol[1];
}
startRowIndex = startRowIndex2;
maxColIndex = Math.max(maxColIndex, startColumnIndex2);
}
return new int[] { startRowIndex, maxColIndex };
} else {
return new int[] {
startRowIndex,
startColumnIndex + countColumns(extNode, null, new ArrayList<String>(), new ArrayList<String>())
};
}
}
}
protected int[] collectResult(
List<Object[]> rows,
JSONArray subProperties,
JSONObject resultNode,
int startRowIndex,
int startColumnIndex,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
int maxStartRowIndex = startRowIndex;
int k = subProperties.length();
for (int c = 0; c < k; c++) {
int[] rowcol = collectResult(
rows,
subProperties.getJSONObject(c),
resultNode,
startRowIndex,
startColumnIndex,
reconCandidateMap
);
maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]);
startColumnIndex = rowcol[1];
}
return new int[] { maxStartRowIndex, startColumnIndex };
}
static protected InputStream doMqlRead(String query) throws IOException {
URL url = new URL("http://api.freebase.com/api/service/mqlread");
URLConnection connection = url.openConnection();
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
try {
String body = "extended=1&query=" + ParsingUtilities.encode(query);
dos.writeBytes(body);
} finally {
dos.flush();
dos.close();
}
connection.connect();
return connection.getInputStream();
}
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
jsonWriter.key("id"); jsonWriter.value(null);
jsonWriter.key("id|=");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
}
static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
String propertyID = node.getString("id");
String expectedTypeID = node.getJSONObject("expected").getString("id");
writer.key(propertyID);
writer.array();
{
if (!expectedTypeID.startsWith("/type/")) { // not literal
writer.object();
writer.key("optional"); writer.value(true);
boolean hasLimit = false;
if (node.has("constraints") && !node.isNull("constraints")) {
JSONObject constraints = node.getJSONObject("constraints");
String[] names = JSONObject.getNames(constraints);
for (String name : names) {
Object value = constraints.get(name);
if (name.equals("limit")) {
hasLimit = true;
}
if (!name.contains(":") &&
!name.equals("limit") &&
!name.equals("optional") &&
!name.equals("count") &&
!name.equals("estimate-count") &&
!name.equals("sort") &&
!name.equals("return")) {
if (name.startsWith("!")) {
name = "!c:" + name.substring(1);
} else {
name = "c:" + name;
}
}
writer.key(name);
writer.value(value);
}
}
if (!hasLimit) {
writer.key("limit"); writer.value(10);
}
{
boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
writer.key("name"); writer.value(null);
writer.key("id"); writer.value(null);
writer.key("type"); writer.array(); writer.endArray();
}
if (hasSubProperties) {
formulateQueryNode(node.getJSONArray("properties"), writer);
}
}
writer.endObject();
}
}
writer.endArray();
}
static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
int l = propertiesA.length();
for (int i = 0; i < l; i++) {
formulateQueryNode(propertiesA.getJSONObject(i), writer);
}
}
static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
String name = obj.getString("name");
List<String> names2 = null;
List<String> path2 = null;
if (columns != null) {
names2 = new ArrayList<String>(names);
names2.add(name);
path2 = new ArrayList<String>(path);
path2.add(obj.getString("id"));
}
if (obj.has("properties") && !obj.isNull("properties")) {
boolean included = (obj.has("included") && obj.getBoolean("included"));
if (included && columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return (included ? 1 : 0) +
countColumns(obj.getJSONArray("properties"), columns, names2, path2);
} else {
if (columns != null) {
JSONObject expected = obj.getJSONObject("expected");
columns.add(new ColumnInfo(names2, path2,
new FreebaseType(expected.getString("id"), expected.getString("name"))));
}
return 1;
}
}
static protected int countColumns(JSONArray a, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
int c = 0;
int l = a.length();
for (int i = 0; i < l; i++) {
c += countColumns(a.getJSONObject(i), columns, names, path);
}
return c;
}
}

223
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java Normal file
View File

@ -0,0 +1,223 @@
package com.google.refine.freebase.util;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.exception.OAuthCommunicationException;
import oauth.signpost.exception.OAuthExpectationFailedException;
import oauth.signpost.exception.OAuthMessageSignerException;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider;
public class FreebaseUtils {
static final public String FREEBASE_HOST = "www.freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
static final private String AGENT_ID = "/en/google_refine";
private static String getUserInfoURL(String host) {
return "http://" + host + "/api/service/user_info";
}
private static String getMQLWriteURL(String host) {
return "http://" + host + "/api/service/mqlwrite";
}
private static String getMQLReadURL(String host) {
return "http://" + host + "/api/service/mqlread";
}
private static String getUserAgent() {
return RefineServlet.FULLNAME;
}
public static String getUserInfo(Credentials credentials, Provider provider)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// sign the request with the oauth library
consumer.sign(httpRequest);
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
public static String getUserBadges(Provider provider, String user_id)
throws ClientProtocolException, IOException, JSONException {
String query = "{" +
"'id' : '" + user_id + "'," +
"'!/type/usergroup/member' : [{" +
"'id' : null," +
"'key' : [{" +
"'namespace' : null" +
"}]" +
"}]" +
"}".replace("'", "\"");
return mqlread(provider, query);
}
public static String mqlread(Provider provider, String query)
throws ClientProtocolException, IOException, JSONException {
JSONObject envelope = new JSONObject();
envelope.put("query", new JSONObject(query));
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("query", envelope.toString()));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
public static String mqlwrite(Credentials credentials, Provider provider, String query)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
JSONObject envelope = new JSONObject();
envelope.put("query", new JSONObject(query));
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("query", envelope.toString()));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
// sign the request with the oauth library
consumer.sign(httpRequest);
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
}
public static String uploadTriples(
HttpServletRequest request,
String qa,
String source_name,
String source_id,
String mdo_id,
String triples
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
JSONObject mdo_info = new JSONObject();
mdo_info.put("name", source_name);
if (source_id != null) {
mdo_info.put("info_source",source_id);
}
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
if (user_info.has("username")) {
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("check_params", "false"));
if (mdo_id != null) {
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
}
if (Boolean.parseBoolean(qa)) {
formparams.add(new BasicNameValuePair("rabj", "true"));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
}
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
} else {
throw new RuntimeException("Invalid credentials");
}
}
static public String getFreeQUrl() {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL;
}
}