').appendTo(divSearch));
-
- MenuSystem.showMenu(menu, function(){});
- MenuSystem.positionMenuAboveBelow(menu, $(elmt));
-
- var suggestOptions = {
- filter : '(all type:/type/property)'
- };
- var sourceTypeID = this._parentUINode.getExpectedType();
- if (sourceTypeID !== null) {
- suggestOptions.filter = '(all type:/type/property (should namespace:' + sourceTypeID + '))'
-
- }
- input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
-
- input[0].focus();
-};
-
-SchemaAlignmentDialog.UILink.prototype.getJSON = function() {
- if ("property" in this._link && this._link.property !== null &&
- "target" in this._link && this._link.target !== null) {
-
- var targetJSON = this._targetUI.getJSON();
- if (targetJSON !== null) {
- var json = {
- property: cloneDeep(this._link.property),
- target: targetJSON
- };
- if (this._link.condition) {
- json.condition = cloneDeep(this._link.condition);
- }
- return json;
- }
- }
- return null;
-};
-
-SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
- var self = this;
- var dismissBusy = DialogSystem.showBusy();
-
- $.getJSON(
- "https://www.googleapis.com/freebase/v1/mqlread?key="+ Freebase.API_KEY + "&query=" + JSON.stringify({
- "id" : this._link.property.id,
- "type" : "/type/property",
- "expected_type" : {
- "id" : null,
- "name" : null,
- "/freebase/type_hints/mediator" : null
- }
- }) + "&callback=?",
- null,
- function(o) {
- dismissBusy();
-
- if ("result" in o) {
- var expected_type = o.result.expected_type;
- self._link.target.type = {
- id: expected_type.id,
- name: expected_type.name
- };
- if (expected_type["/freebase/type_hints/mediator"] === true) {
- self._link.target.nodeType = "anonymous";
- } else if (expected_type.id == "/type/key") {
- self._link.target.nodeType = "cell-as-key";
- } else if (expected_type.id.match(/^\/type\//)) {
- self._link.target.nodeType = "cell-as-value";
- } else if (!("topic" in self._link.target)) {
- self._link.target.nodeType = "cell-as-topic";
- self._link.target.createForNoReconMatch = true;
- }
-
- self._targetUI.render();
- }
-
- self._renderMain();
- self._dialog.preview();
- },
- "jsonp"
- );
-};
diff --git a/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
deleted file mode 100644
index 354ca3e7c..000000000
--- a/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
+++ /dev/null
@@ -1,754 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- */
-
-SchemaAlignmentDialog.UINode = function(dialog, node, table, options) {
- this._dialog = dialog;
- this._node = node;
- this._options = options;
-
- if ("columnName" in this._node) {
- this._node.columnNames = [ this._node.columnName ];
- delete this._node.columnName;
- }
-
- this._linkUIs = [];
- this._detailsRendered = false;
-
- this._tr = table.insertRow(table.rows.length);
- this._tdMain = this._tr.insertCell(0);
- this._tdToggle = this._tr.insertCell(1);
- this._tdDetails = this._tr.insertCell(2);
-
- $(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded");
- $(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide();
- $(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide();
-
- this._renderMain();
-
- this._expanded = options.expanded;
- if (this._isExpandable()) {
- this._showExpandable();
- }
-};
-
-SchemaAlignmentDialog.UINode.prototype.dispose = function() {
- // nothing for now
-};
-
-SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) {
- for (var i = this._linkUIs.length - 1; i >= 0; i--) {
- if (this._linkUIs[i] === linkUI) {
- this._linkUIs.splice(i, 1);
- this._node.links.splice(i, 1);
- break;
- }
- }
-};
-
-SchemaAlignmentDialog.UINode.prototype._isExpandable = function() {
- return this._node.nodeType == "cell-as-topic" ||
- this._node.nodeType == "anonymous" ||
- this._node.nodeType == "topic";
-};
-
-SchemaAlignmentDialog.UINode.prototype.render = function() {
- this._renderMain();
- if (this._isExpandable()) {
- this._showExpandable();
- } else {
- this._hideExpandable();
- }
-};
-
-SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() {
- if ("type" in this._node) {
- return this._node.type.id;
- }
- return null;
-};
-
-SchemaAlignmentDialog.UINode.prototype._renderMain = function() {
- $(this._tdMain).empty();
-
- var self = this;
- var a = $('
')
- .addClass("schema-alignment-node-tag")
- .appendTo(this._tdMain)
- .click(function(evt) {
- self._showNodeConfigDialog();
- });
-
- if (this._node.nodeType == "cell-as-topic" ||
- this._node.nodeType == "cell-as-value" ||
- this._node.nodeType == "cell-as-key") {
-
- if ("columnNames" in this._node) {
- for (var c = 0; c < this._node.columnNames.length; c++) {
- if (c > 0) {
- $('
').text(", ").appendTo(a);
- }
-
- $('')
- .text(this._node.columnNames[c])
- .addClass("schema-alignment-node-column")
- .appendTo(a);
- }
-
- $('').text(
- this._node.columnNames.length > 1 ? " "
- + $.i18n._('fb-schema-alignment')["cells"] : " "
- + $.i18n._('fb-schema-alignment')["cell"]).appendTo(a);
- } else {
- a.html(this._options.mustBeCellTopic ? $.i18n._('fb-schema-alignment')["which-column"] : $.i18n._('fb-schema-alignment')["configure"]);
- }
- } else if (this._node.nodeType == "topic") {
- if ("topic" in this._node) {
- a.html(this._node.topic.name);
- } else if ("id" in this._node) {
- a.html(this._node.topic.id);
- } else {
- a.html($.i18n._('fb-schema-alignment')["which-topic"]);
- }
- } else if (this._node.nodeType == "value") {
- if ("value" in this._node) {
- a.html(this._node.value);
- } else {
- a.html($.i18n._('fb-schema-alignment')["what-value"]);
- }
- } else if (this._node.nodeType == "anonymous") {
- a.html("("+$.i18n._('fb-schema-alignment')["anonymous"]+")");
- }
-};
-
-SchemaAlignmentDialog.UINode.prototype._showExpandable = function() {
- $(this._tdToggle).show();
- $(this._tdDetails).show();
-
- if (this._detailsRendered) {
- return;
- }
- this._detailsRendered = true;
-
- this._collapsedDetailDiv = $('
').appendTo(this._tdDetails).addClass("padded").html("...");
- this._expandedDetailDiv = $('
').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
-
- this._renderDetails();
-
- var self = this;
- var show = function() {
- if (self._expanded) {
- self._collapsedDetailDiv.hide();
- self._expandedDetailDiv.show();
- } else {
- self._collapsedDetailDiv.show();
- self._expandedDetailDiv.hide();
- }
- };
- show();
-
- $(this._tdToggle).html(" ");
- $('
').addClass("padded").appendTo(this._expandedDetailDiv);
-
- $('
').addClass("dialog-header").text($.i18n._('fb-schema-alignment')["skeleton-node"]).appendTo(frame);
- var body = $('
').addClass("dialog-body").appendTo(frame);
- var footer = $('
').addClass("dialog-footer").appendTo(frame);
-
- /*--------------------------------------------------
- * Body
- *--------------------------------------------------
- */
- var literalTypeSelectHtml =
- ''+$.i18n._('fb-schema-alignment')["text"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["int"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["float"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["double"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["boolean"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["date-time"]+' ' +
- ''+$.i18n._('fb-schema-alignment')["rawstring"]+' ';
-
- var html = $(
- '' +
- '' +
- '' +
- '' +
- ' ' +
-
- '' +
- '' +
- ' ' +
- ' ' +
- '
topicNames = new ArrayList();
- List topicIds = new ArrayList();
- Set ids = new HashSet();
- for (int i = 0; i < length; i++) {
- int rowIndex = rowIndices.getInt(i);
- if (rowIndex >= 0 && rowIndex < project.rows.size()) {
- Row row = project.rows.get(rowIndex);
- Cell cell = row.getCell(cellIndex);
- if (cell != null && cell.recon != null && cell.recon.match != null) {
- topicNames.add(cell.recon.match.name);
- topicIds.add(cell.recon.match.id);
- ids.add(cell.recon.match.id);
- } else {
- topicNames.add(null);
- topicIds.add(null);
- ids.add(null);
- }
- }
- }
-
- Map reconCandidateMap = new HashMap();
- FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json);
- Map map = job.extend(ids, reconCandidateMap);
-
- response.setCharacterEncoding("UTF-8");
- response.setHeader("Content-Type", "application/json");
-
- JSONWriter writer = new JSONWriter(response.getWriter());
- writer.object();
- writer.key("code"); writer.value("ok");
- writer.key("columns");
- writer.array();
- for (ColumnInfo info : job.columns) {
- writer.object();
- writer.key("names");
- writer.array();
- for (String name : info.names) {
- writer.value(name);
- }
- writer.endArray();
- writer.key("path");
- writer.array();
- for (String id : info.path) {
- writer.value(id);
- }
- writer.endArray();
- writer.endObject();
- }
- writer.endArray();
-
- writer.key("rows");
- writer.array();
- for (int r = 0; r < topicNames.size(); r++) {
- String id = topicIds.get(r);
- String topicName = topicNames.get(r);
-
- if (id != null && map.containsKey(id)) {
- DataExtension ext = map.get(id);
- boolean first = true;
-
- if (ext.data.length > 0) {
- for (Object[] row : ext.data) {
- writer.array();
- if (first) {
- writer.value(topicName);
- first = false;
- } else {
- writer.value(null);
- }
-
- for (Object cell : row) {
- if (cell != null && cell instanceof ReconCandidate) {
- ReconCandidate rc = (ReconCandidate) cell;
- writer.object();
- writer.key("id"); writer.value(rc.id);
- writer.key("name"); writer.value(rc.name);
- writer.endObject();
- } else {
- writer.value(cell);
- }
- }
-
- writer.endArray();
- }
- continue;
- }
- }
-
- writer.array();
- if (id != null) {
- writer.object();
- writer.key("id"); writer.value(id);
- writer.key("name"); writer.value(topicName);
- writer.endObject();
- } else {
- writer.value("");
- }
- writer.endArray();
- }
- writer.endArray();
-
- writer.endObject();
- } catch (Exception e) {
- respondException(response, e);
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java
deleted file mode 100644
index 2f51075b8..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.commands;
-
-import java.io.IOException;
-import java.io.StringWriter;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONObject;
-
-import com.google.refine.browsing.Engine;
-import com.google.refine.browsing.FilteredRows;
-import com.google.refine.commands.Command;
-import com.google.refine.freebase.protograph.Protograph;
-import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
-import com.google.refine.freebase.protograph.transpose.Transposer;
-import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
-import com.google.refine.model.Project;
-import com.google.refine.util.ParsingUtilities;
-
-public class PreviewProtographCommand extends Command {
- @Override
- public void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- try {
- Project project = getProject(request);
- Engine engine = getEngine(request, project);
- FilteredRows filteredRows = engine.getAllFilteredRows();
-
- response.setCharacterEncoding("UTF-8");
- response.setHeader("Content-Type", "application/json");
-
- String jsonString = request.getParameter("protograph");
- JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
- Protograph protograph = Protograph.reconstruct(json);
-
- StringBuffer sb = new StringBuffer(2048);
- sb.append("{ ");
-
- {
- StringWriter stringWriter = new StringWriter();
- TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter);
-
- Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
- nodeFactory.flush();
-
- sb.append("\"tripleloader\" : ");
- sb.append(JSONObject.quote(stringWriter.toString()));
- }
-
- {
- StringWriter stringWriter = new StringWriter();
- MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter);
-
- Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
- nodeFactory.flush();
-
- sb.append(", \"mqllike\" : ");
- sb.append(stringWriter.toString());
- }
-
- sb.append(" }");
-
- respond(response, sb.toString());
- } catch (Exception e) {
- respondException(response, e);
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java
deleted file mode 100644
index cec61e92e..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.commands;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONObject;
-
-import com.google.refine.commands.Command;
-import com.google.refine.freebase.operations.SaveProtographOperation;
-import com.google.refine.freebase.protograph.Protograph;
-import com.google.refine.model.AbstractOperation;
-import com.google.refine.model.Project;
-import com.google.refine.process.Process;
-import com.google.refine.util.ParsingUtilities;
-
-public class SaveProtographCommand extends Command {
- @Override
- public void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- try {
- Project project = getProject(request);
-
- String jsonString = request.getParameter("protograph");
- JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
- Protograph protograph = Protograph.reconstruct(json);
-
- AbstractOperation op = new SaveProtographOperation(protograph);
- Process process = op.createProcess(project, new Properties());
-
- performProcessAndRespond(request, response, project, process);
- } catch (Exception e) {
- respondException(response, e);
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java
deleted file mode 100644
index c92fef314..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.commands;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.browsing.Engine;
-import com.google.refine.commands.Command;
-import com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter;
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.model.Project;
-import com.google.refine.preference.PreferenceStore;
-import com.google.refine.util.ParsingUtilities;
-
-public class UploadDataCommand extends Command {
- final static public String s_dataLoadJobIDPref = "freebase.load.jobID";
- final static public String s_dataLoadJobNamePref = "freebase.load.jobName";
-
- @Override
- public void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- ProjectManager.singleton.setBusy(true);
- try {
- Project project = getProject(request);
- Engine engine = getEngine(request, project);
- PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore();
-
- TripleLoaderExporter exporter = new TripleLoaderExporter();
- StringWriter triples = new StringWriter(10 * 1024 * 1024);
- exporter.export(project, new Properties(), engine, triples);
-
- String source_name = request.getParameter("source_name");
- String source_id = request.getParameter("source_id");
- String qa = request.getParameter("qa");
- String mdo_id = null;
-
- preferenceStore.put(s_dataLoadJobNamePref, source_name);
-
- try {
- Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref);
- if (jobID != null) {
- URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID);
- String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
-
- if (!s.equals("null")) {
- mdo_id = s;
- }
- }
- } catch (Exception e) {
- // ignore
- }
-
- String uploadResponse = FreebaseUtils.uploadTriples(
- request, qa, source_name, source_id, mdo_id, triples.toString()
- );
-
- response.setCharacterEncoding("UTF-8");
- response.setHeader("Content-Type", "application/json");
-
- try {
- JSONObject obj = new JSONObject(uploadResponse);
- if (obj.has("result") && !obj.isNull("result")) {
- JSONObject result = obj.getJSONObject("result");
- if (result.has("job_id") && !result.isNull("job_id")) {
- Integer jobID = result.getInt("job_id");
- project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
- }
- }
- response.getWriter().write(uploadResponse);
- } catch (JSONException e) {
- respond(response,"500 Error", uploadResponse);
- }
- } catch (Exception e) {
- respondException(response, e);
- } finally {
- ProjectManager.singleton.setBusy(false);
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java
deleted file mode 100644
index 990f71f1a..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.commands.auth;
-
-import java.io.IOException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.refine.commands.Command;
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.oauth.Credentials;
-import com.google.refine.oauth.OAuthUtilities;
-import com.google.refine.oauth.Provider;
-
-public class CheckAuthorizationCommand extends Command {
-
- final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
-
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-
- try {
- Provider provider = OAuthUtilities.getProvider(request);
-
- // this cookie should not be there, but this is good hygiene practice
- Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
-
- Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
-
- response.setCharacterEncoding("UTF-8");
- response.setHeader("Content-Type", "application/json");
-
- if (access_credentials != null) {
- String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
- response.getWriter().write(user_info);
- } else {
- respond(response, "401 Unauthorized", "You don't have the right credentials");
- }
- } catch (Exception e) {
- logger.info("error",e);
- respondException(response, e);
- }
- }
-
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java
deleted file mode 100644
index 6c808d45a..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.commands.auth;
-
-import java.io.IOException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.refine.commands.Command;
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.oauth.OAuthUtilities;
-import com.google.refine.oauth.Provider;
-
-public class GetUserBadgesCommand extends Command {
-
- final static Logger logger = LoggerFactory.getLogger("get-version_command");
-
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-
- try {
- Provider provider = OAuthUtilities.getProvider(request);
- String user_id = request.getParameter("user_id");
- response.setCharacterEncoding("UTF-8");
- response.setHeader("Content-Type", "application/json");
- String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
- response.getWriter().write(user_badges);
- } catch (Exception e) {
- logger.info("error",e);
- respondException(response, e);
- }
- }
-
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyQuote.java b/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyQuote.java
deleted file mode 100644
index 653e61f66..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyQuote.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.expr;
-
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.google.refine.grel.Function;
-
-public class MqlKeyQuote implements Function {
- final static private String keyStartChar = "A-Za-z0-9";
- final static private String keyInternalChar = "A-Za-z0-9_-";
- final static private String keyEndChar = "A-Za-z0-9_";
- final static private String fullValidKey = "^[" + keyStartChar + "][" + keyInternalChar + "]*[" + keyEndChar + "]$";
- final static private String keyCharMustQuote = "([^" + keyInternalChar + "])";
-
- final static private Pattern fullValidKeyPattern = Pattern.compile(fullValidKey);
- final static private Pattern keyCharMustQuotePattern = Pattern.compile(keyCharMustQuote);
-
- @Override
- public Object call(Properties bindings, Object[] args) {
- if (args.length == 1) {
- Object o1 = args[0];
- if (o1 != null && o1 instanceof String) {
- return mqlKeyQuote((String) o1);
- }
- }
- return null;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("description"); writer.value("Unquotes a MQL key");
- writer.key("params"); writer.value("string s");
- writer.key("returns"); writer.value("string");
- writer.endObject();
- }
-
- static String mqlKeyQuote(String s) {
- if (fullValidKeyPattern.matcher(s).find()) {
- return s;
- }
-
- StringBuffer sb = new StringBuffer();
-
- int last = 0;
- Matcher m = keyCharMustQuotePattern.matcher(s);
- while (m.find()) {
- int start = m.start();
- int end = m.end();
- if (start > last) {
- sb.append(s.substring(last, start));
- }
- last = end;
-
- sb.append('$');
- sb.append(quote(s.charAt(start)));
- }
-
- if (last < s.length()) {
- sb.append(s.substring(last));
- }
-
- if (sb.length() > 0) {
- if (sb.charAt(0) == '-' || sb.charAt(0) == '_') {
- char c = sb.charAt(0);
- sb.deleteCharAt(0);
- sb.insert(0, '$');
- sb.insert(1, quote(c));
- }
-
- int length = sb.length();
- if (sb.charAt(length-1) == '-') {
- sb.deleteCharAt(length-1);
- sb.insert(length-1, '$');
- sb.insert(length, quote('-'));
- }
- }
-
- return sb.toString();
- }
-
- static String quote(char c) {
- return StringUtils.leftPad(Integer.toHexString(c).toUpperCase(), 4, '0');
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyUnquote.java b/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyUnquote.java
deleted file mode 100644
index 479c3da77..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/expr/MqlKeyUnquote.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.expr;
-
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.google.refine.grel.Function;
-
-public class MqlKeyUnquote implements Function {
- final static private Pattern quotedCharPattern = Pattern.compile("\\$([0-9A-Fa-f]{4})");
-
- @Override
- public Object call(Properties bindings, Object[] args) {
- if (args.length == 1) {
- Object o1 = args[0];
- if (o1 != null && o1 instanceof String) {
- String s = (String) o1;
- StringBuffer sb = new StringBuffer();
-
- int last = 0;
- Matcher m = quotedCharPattern.matcher(s);
- while (m.find()) {
- int start = m.start();
- int end = m.end();
- if (start > last) {
- sb.append(s.substring(last, start));
- }
- last = end;
-
- sb.append((char)Integer.parseInt(s.substring(start + 1, end), 16));
- }
-
- if (last < s.length()) {
- sb.append(s.substring(last));
- }
-
- return sb.toString();
- }
- }
- return null;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("description"); writer.value("Quotes a string into a MQL key");
- writer.key("params"); writer.value("string s");
- writer.key("returns"); writer.value("string");
- writer.endObject();
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java b/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java
deleted file mode 100644
index 14ba54dc5..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.changes;
-
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.io.Serializable;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseType;
-import com.google.refine.freebase.model.recon.DataExtensionReconConfig;
-import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
-import com.google.refine.history.Change;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
-import com.google.refine.model.ModelException;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.model.ReconStats;
-import com.google.refine.model.Row;
-import com.google.refine.util.ParsingUtilities;
-import com.google.refine.util.Pool;
-
-public class DataExtensionChange implements Change {
- final protected String _baseColumnName;
- final protected int _columnInsertIndex;
-
- final protected List _columnNames;
- final protected List _columnTypes;
-
- final protected List _rowIndices;
- final protected List _dataExtensions;
-
- protected long _historyEntryID;
- protected int _firstNewCellIndex = -1;
- protected List _oldRows;
- protected List _newRows;
-
- public DataExtensionChange(
- String baseColumnName,
- int columnInsertIndex,
- List columnNames,
- List columnTypes,
- List rowIndices,
- List dataExtensions,
- long historyEntryID
- ) {
- _baseColumnName = baseColumnName;
- _columnInsertIndex = columnInsertIndex;
-
- _columnNames = columnNames;
- _columnTypes = columnTypes;
-
- _rowIndices = rowIndices;
- _dataExtensions = dataExtensions;
-
- _historyEntryID = historyEntryID;
- }
-
- protected DataExtensionChange(
- String baseColumnName,
- int columnInsertIndex,
-
- List columnNames,
- List columnTypes,
-
- List rowIndices,
- List dataExtensions,
- int firstNewCellIndex,
- List oldRows,
- List newRows
- ) {
- _baseColumnName = baseColumnName;
- _columnInsertIndex = columnInsertIndex;
-
- _columnNames = columnNames;
- _columnTypes = columnTypes;
-
- _rowIndices = rowIndices;
- _dataExtensions = dataExtensions;
-
- _firstNewCellIndex = firstNewCellIndex;
- _oldRows = oldRows;
- _newRows = newRows;
- }
-
- @Override
- public void apply(Project project) {
- synchronized (project) {
- if (_firstNewCellIndex < 0) {
- _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
- for (int i = 1; i < _columnNames.size(); i++) {
- project.columnModel.allocateNewCellIndex();
- }
-
- _oldRows = new ArrayList(project.rows);
-
- _newRows = new ArrayList(project.rows.size());
-
- int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
- int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
- int index = 0;
-
- int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
- DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
-
- index++;
-
- Map reconMap = new HashMap();
-
- for (int r = 0; r < _oldRows.size(); r++) {
- Row oldRow = _oldRows.get(r);
- if (r < rowIndex) {
- _newRows.add(oldRow.dup());
- continue;
- }
-
- if (dataExtension == null || dataExtension.data.length == 0) {
- _newRows.add(oldRow);
- } else {
- Row firstNewRow = oldRow.dup();
- extendRow(firstNewRow, dataExtension, 0, reconMap);
- _newRows.add(firstNewRow);
-
- int r2 = r + 1;
- for (int subR = 1; subR < dataExtension.data.length; subR++) {
- if (r2 < project.rows.size()) {
- Row oldRow2 = project.rows.get(r2);
- if (oldRow2.isCellBlank(cellIndex) &&
- oldRow2.isCellBlank(keyCellIndex)) {
-
- Row newRow = oldRow2.dup();
- extendRow(newRow, dataExtension, subR, reconMap);
-
- _newRows.add(newRow);
- r2++;
-
- continue;
- }
- }
-
- Row newRow = new Row(cellIndex + _columnNames.size());
- extendRow(newRow, dataExtension, subR, reconMap);
-
- _newRows.add(newRow);
- }
-
- r = r2 - 1; // r will be incremented by the for loop anyway
- }
-
- rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
- dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
- index++;
- }
- }
-
- project.rows.clear();
- project.rows.addAll(_newRows);
-
- for (int i = 0; i < _columnNames.size(); i++) {
- String name = _columnNames.get(i);
- int cellIndex = _firstNewCellIndex + i;
-
- Column column = new Column(cellIndex, name);
- column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
- column.setReconStats(ReconStats.create(project, cellIndex));
-
- try {
- project.columnModel.addColumn(_columnInsertIndex + i, column, true);
-
- // the column might have been renamed to avoid collision
- _columnNames.set(i, column.getName());
- } catch (ModelException e) {
- // won't get here since we set the avoid collision flag
- }
- }
-
- project.update();
- }
- }
-
- protected void extendRow(
- Row row,
- DataExtension dataExtension,
- int extensionRowIndex,
- Map reconMap
- ) {
- Object[] values = dataExtension.data[extensionRowIndex];
- for (int c = 0; c < values.length; c++) {
- Object value = values[c];
- Cell cell = null;
-
- if (value instanceof ReconCandidate) {
- ReconCandidate rc = (ReconCandidate) value;
- Recon recon;
- if (reconMap.containsKey(rc.id)) {
- recon = reconMap.get(rc.id);
- } else {
- recon = Recon.makeFreebaseRecon(_historyEntryID);
- recon.addCandidate(rc);
- recon.service = "mql";
- recon.match = rc;
- recon.matchRank = 0;
- recon.judgment = Judgment.Matched;
- recon.judgmentAction = "auto";
- recon.judgmentBatchSize = 1;
-
- reconMap.put(rc.id, recon);
- }
- cell = new Cell(rc.name, recon);
- } else {
- cell = new Cell((Serializable) value, null);
- }
-
- row.setCell(_firstNewCellIndex + c, cell);
- }
- }
-
- @Override
- public void revert(Project project) {
- synchronized (project) {
- project.rows.clear();
- project.rows.addAll(_oldRows);
-
- for (int i = 0; i < _columnNames.size(); i++) {
- project.columnModel.columns.remove(_columnInsertIndex);
- }
-
- project.update();
- }
- }
-
- @Override
- public void save(Writer writer, Properties options) throws IOException {
- writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
- writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
- writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
- for (String name : _columnNames) {
- writer.write(name); writer.write('\n');
- }
- writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
- for (FreebaseType type : _columnTypes) {
- try {
- JSONWriter jsonWriter = new JSONWriter(writer);
-
- type.write(jsonWriter, options);
- } catch (JSONException e) {
- // ???
- }
- writer.write('\n');
- }
- writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
- for (Integer rowIndex : _rowIndices) {
- writer.write(rowIndex.toString()); writer.write('\n');
- }
- writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
- for (DataExtension dataExtension : _dataExtensions) {
- if (dataExtension == null) {
- writer.write('\n');
- continue;
- }
-
- writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
-
- for (Object[] values : dataExtension.data) {
- for (Object value : values) {
- if (value == null) {
- writer.write("null");
- } else if (value instanceof ReconCandidate) {
- try {
- JSONWriter jsonWriter = new JSONWriter(writer);
- ((ReconCandidate) value).write(jsonWriter, options);
- } catch (JSONException e) {
- // ???
- }
- } else if (value instanceof String) {
- writer.write(JSONObject.quote((String) value));
- } else {
- writer.write(value.toString());
- }
- writer.write('\n');
- }
- }
- }
-
- writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
-
- writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
- for (Row row : _newRows) {
- row.save(writer, options);
- writer.write('\n');
- }
- writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
- for (Row row : _oldRows) {
- row.save(writer, options);
- writer.write('\n');
- }
- writer.write("/ec/\n"); // end of change marker
- }
-
- static public Change load(LineNumberReader reader, Pool pool) throws Exception {
- String baseColumnName = null;
- int columnInsertIndex = -1;
-
- List columnNames = null;
- List columnTypes = null;
-
- List rowIndices = null;
- List dataExtensions = null;
-
- List oldRows = null;
- List newRows = null;
-
- int firstNewCellIndex = -1;
-
- String line;
- while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
- int equal = line.indexOf('=');
- CharSequence field = line.subSequence(0, equal);
- String value = line.substring(equal + 1);
-
- if ("baseColumnName".equals(field)) {
- baseColumnName = value;
- } else if ("columnInsertIndex".equals(field)) {
- columnInsertIndex = Integer.parseInt(value);
- } else if ("firstNewCellIndex".equals(field)) {
- firstNewCellIndex = Integer.parseInt(value);
- } else if ("rowIndexCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- rowIndices = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
- if (line != null) {
- rowIndices.add(Integer.parseInt(line));
- }
- }
- } else if ("columnNameCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- columnNames = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
- if (line != null) {
- columnNames.add(line);
- }
- }
- } else if ("columnTypeCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- columnTypes = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
- columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
- }
- } else if ("dataExtensionCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- dataExtensions = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
-
- if (line == null) {
- continue;
- }
-
- if (line.length() == 0) {
- dataExtensions.add(null);
- continue;
- }
-
- int rowCount = Integer.parseInt(line);
- Object[][] data = new Object[rowCount][];
-
- for (int r = 0; r < rowCount; r++) {
- Object[] row = new Object[columnNames.size()];
- for (int c = 0; c < columnNames.size(); c++) {
- line = reader.readLine();
-
- row[c] = ReconCandidate.loadStreaming(line);
- }
-
- data[r] = row;
- }
-
- dataExtensions.add(new DataExtension(data));
- }
- } else if ("oldRowCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- oldRows = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
- if (line != null) {
- oldRows.add(Row.load(line, pool));
- }
- }
- } else if ("newRowCount".equals(field)) {
- int count = Integer.parseInt(value);
-
- newRows = new ArrayList(count);
- for (int i = 0; i < count; i++) {
- line = reader.readLine();
- if (line != null) {
- newRows.add(Row.load(line, pool));
- }
- }
- }
-
- }
-
- DataExtensionChange change = new DataExtensionChange(
- baseColumnName,
- columnInsertIndex,
- columnNames,
- columnTypes,
- rowIndices,
- dataExtensions,
- firstNewCellIndex,
- oldRows,
- newRows
- );
-
-
- return change;
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java
deleted file mode 100644
index f334fe0c3..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.recon;
-
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseType;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Row;
-import com.google.refine.model.recon.ReconConfig;
-import com.google.refine.model.recon.ReconJob;
-
-public class DataExtensionReconConfig extends StrictReconConfig {
- final public FreebaseType type;
-
- private final static String WARN = "Not implemented";
-
- static public ReconConfig reconstruct(JSONObject obj) throws Exception {
- JSONObject type = obj.getJSONObject("type");
-
- return new DataExtensionReconConfig(
- new FreebaseType(
- type.getString("id"),
- type.getString("name")
- )
- );
- }
-
- public DataExtensionReconConfig(FreebaseType type) {
- this.type = type;
- }
-
- @Override
- public ReconJob createJob(Project project, int rowIndex, Row row,
- String columnName, Cell cell) {
- throw new RuntimeException(WARN);
- }
-
- @Override
- public int getBatchSize() {
- throw new RuntimeException(WARN);
- }
-
- @Override
- public void write(JSONWriter writer, Properties options) throws JSONException {
- writer.object();
- writer.key("mode"); writer.value("extend");
- writer.key("type"); type.write(writer, options);
- writer.endObject();
- }
-
- @Override
- public List batchRecon(List jobs, long historyEntryID) {
- throw new RuntimeException(WARN);
- }
-
- @Override
- public String getBriefDescription(Project project, String columnName) {
- throw new RuntimeException(WARN);
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
deleted file mode 100644
index 8fa83fc96..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
-
-Copyright 2010,2013 Google Inc. and other contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.recon;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.model.Row;
-import com.google.refine.model.recon.ReconConfig;
-import com.google.refine.model.recon.ReconJob;
-import com.google.refine.util.ParsingUtilities;
-
-public class GuidBasedReconConfig extends StrictReconConfig {
- static public ReconConfig reconstruct(JSONObject obj) throws Exception {
- return new GuidBasedReconConfig();
- }
-
- public GuidBasedReconConfig() {
- }
-
- static protected class GuidBasedReconJob extends ReconJob {
- String guid;
-
- @Override
- public String getStringKey() {
- return guid;
- }
- }
-
- @Override
- public ReconJob createJob(Project project, int rowIndex, Row row,
- String columnName, Cell cell) {
-
- GuidBasedReconJob job = new GuidBasedReconJob();
- String s = cell.value.toString();
-
- if (s.startsWith("/guid/")) {
- s = "#" + s.substring(6);
- } else if (!s.startsWith("#")) {
- s = "#" + s;
- }
-
- job.guid = s;
-
- return job;
- }
-
- @Override
- public int getBatchSize() {
- return 50;
- }
-
- @Override
- public String getBriefDescription(Project project, String columnName) {
- return "Reconcile cells in column " + columnName + " as Freebase IDs";
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("mode"); writer.value("strict");
- writer.key("match"); writer.value("id");
- writer.endObject();
- }
-
- @Override
- public List batchRecon(List jobs, long historyEntryID) {
- List recons = new ArrayList(jobs.size());
- Map guidToRecon = new HashMap();
-
- try {
- String query = buildQuery(jobs);
-
- String s = FreebaseUtils.mqlread(query);
- JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-
- if (o.has("result")) {
- JSONArray results = o.getJSONArray("result");
- int count = results.length();
-
- for (int i = 0; i < count; i++) {
- JSONObject result = results.getJSONObject(i);
-
- String guid = result.getString("guid");
-
- JSONArray types = result.getJSONArray("type");
- String[] typeIDs = new String[types.length()];
- for (int j = 0; j < typeIDs.length; j++) {
- typeIDs[j] = types.getString(j);
- }
-
- ReconCandidate candidate = new ReconCandidate(
- result.getString("id"),
- result.getString("name"),
- typeIDs,
- 100
- );
-
- Recon recon = Recon.makeFreebaseRecon(historyEntryID);
- recon.addCandidate(candidate);
- recon.service = "mql";
- recon.judgment = Judgment.Matched;
- recon.judgmentAction = "auto";
- recon.match = candidate;
- recon.matchRank = 0;
-
- guidToRecon.put(guid, recon);
- }
- }
- } catch (IOException e) {
- LOGGER.error("IOException during recon : ",e);
- } catch (JSONException e) {
- LOGGER.error("JSONException during recon : ",e);
- }
-
- for (ReconJob job : jobs) {
- String guid = ((GuidBasedReconJob) job).guid;
- Recon recon = guidToRecon.get(guid);
- if (recon == null) {
- recon = createNoMatchRecon(historyEntryID);
- }
- recons.add(recon);
- }
-
- return recons;
- }
-
- private String buildQuery(List jobs)
- throws JSONException {
- String query = null;
-
- StringWriter stringWriter = new StringWriter();
- JSONWriter jsonWriter = new JSONWriter(stringWriter);
-
- jsonWriter.array();
- jsonWriter.object();
-
- jsonWriter.key("id"); jsonWriter.value(null);
- jsonWriter.key("name"); jsonWriter.value(null);
- jsonWriter.key("guid"); jsonWriter.value(null);
- jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
-
- jsonWriter.key("guid|=");
- jsonWriter.array();
- for (ReconJob job : jobs) {
- jsonWriter.value(((GuidBasedReconJob) job).guid);
- }
- jsonWriter.endArray();
-
- jsonWriter.endObject();
- jsonWriter.endArray();
-
- query = stringWriter.toString();
- return query;
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
deleted file mode 100644
index 739a31458..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
-
-Copyright 2010,2013 Google Inc. and other contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.recon;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.model.Row;
-import com.google.refine.model.recon.ReconConfig;
-import com.google.refine.model.recon.ReconJob;
-import com.google.refine.util.ParsingUtilities;
-
-public class IdBasedReconConfig extends StrictReconConfig {
- static public ReconConfig reconstruct(JSONObject obj) throws Exception {
- return new IdBasedReconConfig();
- }
-
- public IdBasedReconConfig() {
- }
-
- static protected class IdBasedReconJob extends ReconJob {
- String id;
-
- @Override
- public String getStringKey() {
- return id;
- }
- }
-
- @Override
- public ReconJob createJob(Project project, int rowIndex, Row row,
- String columnName, Cell cell) {
-
- IdBasedReconJob job = new IdBasedReconJob();
- String s = cell.value.toString();
-
- if (!s.startsWith("/")) {
- if (s.startsWith("92")) {
- s = "/guid/" + s;
- } else if (!s.contains("/")){
- s = "/en/" + s;
- } else {
- s = "/" + s;
- }
- }
-
- job.id = s;
-
- return job;
- }
-
- @Override
- public int getBatchSize() {
- return 40;
- }
-
- @Override
- public String getBriefDescription(Project project, String columnName) {
- return "Reconcile cells in column " + columnName + " as Freebase IDs";
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("mode"); writer.value("strict");
- writer.key("match"); writer.value("id");
- writer.endObject();
- }
-
- @Override
- public List batchRecon(List jobs, long historyEntryID) {
- List recons = new ArrayList(jobs.size());
- Map idToRecon = new HashMap();
-
- try {
- String query = buildQuery(jobs);
- String s = FreebaseUtils.mqlread(query);
-
- JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
- if (o.has("result")) {
- JSONArray results = o.getJSONArray("result");
- int count = results.length();
-
- for (int i = 0; i < count; i++) {
- JSONObject result = results.getJSONObject(i);
-
- String id = result.getString("id");
-
- JSONArray types = result.getJSONArray("type");
- String[] typeIDs = new String[types.length()];
- for (int j = 0; j < typeIDs.length; j++) {
- typeIDs[j] = types.getString(j);
- }
-
- ReconCandidate candidate = new ReconCandidate(
- id,
- result.getString("name"),
- typeIDs,
- 100
- );
-
- Recon recon = Recon.makeFreebaseRecon(historyEntryID);
- recon.addCandidate(candidate);
- recon.service = "mql";
- recon.judgment = Judgment.Matched;
- recon.judgmentAction = "auto";
- recon.match = candidate;
- recon.matchRank = 0;
-
- idToRecon.put(id, recon);
- }
- }
- } catch (IOException e) {
- LOGGER.error("IOException during recon : ",e);
- } catch (JSONException e) {
- LOGGER.error("JSONException during recon : ",e);
- }
-
- for (ReconJob job : jobs) {
- String id = ((IdBasedReconJob) job).id;
- Recon recon = idToRecon.get(id);
- if (recon == null) {
- recon = createNoMatchRecon(historyEntryID);
- }
- recons.add(recon);
- }
-
- return recons;
- }
-
- private String buildQuery(List jobs)
- throws JSONException {
- String query = null;
- {
- StringWriter stringWriter = new StringWriter();
- JSONWriter jsonWriter = new JSONWriter(stringWriter);
-
- jsonWriter.array();
- jsonWriter.object();
-
- jsonWriter.key("id"); jsonWriter.value(null);
- jsonWriter.key("name"); jsonWriter.value(null);
- jsonWriter.key("guid"); jsonWriter.value(null);
- jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
-
- jsonWriter.key("id|=");
- jsonWriter.array();
- for (ReconJob job : jobs) {
- jsonWriter.value(((IdBasedReconJob) job).id);
- }
- jsonWriter.endArray();
-
- jsonWriter.endObject();
- jsonWriter.endArray();
-
- query = stringWriter.toString();
- }
- return query;
- }
-
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
deleted file mode 100644
index c082e9d11..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.recon;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseTopic;
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.model.Row;
-import com.google.refine.model.recon.ReconConfig;
-import com.google.refine.model.recon.ReconJob;
-import com.google.refine.util.ParsingUtilities;
-
-public class KeyBasedReconConfig extends StrictReconConfig {
- final public FreebaseTopic namespace;
-
- static public ReconConfig reconstruct(JSONObject obj) throws Exception {
- JSONObject ns = obj.getJSONObject("namespace");
-
- return new KeyBasedReconConfig(
- new FreebaseTopic(
- ns.getString("id"),
- ns.getString("name")
- )
- );
- }
-
- public KeyBasedReconConfig(FreebaseTopic namespace) {
- this.namespace = namespace;
- }
-
- static protected class KeyBasedReconJob extends ReconJob {
- String key;
-
- @Override
- public String getStringKey() {
- return key;
- }
- }
-
- @Override
- public ReconJob createJob(Project project, int rowIndex, Row row,
- String columnName, Cell cell) {
-
- KeyBasedReconJob job = new KeyBasedReconJob();
-
- job.key = cell.value.toString().replace(' ', '_');
-
- return job;
- }
-
- @Override
- public int getBatchSize() {
- return 40;
- }
-
- @Override
- public String getBriefDescription(Project project, String columnName) {
- return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("mode"); writer.value("strict");
- writer.key("match"); writer.value("key");
- writer.key("namespace"); namespace.write(writer, options);
- writer.endObject();
- }
-
- @Override
- public List batchRecon(List jobs, long historyEntryID) {
- List recons = new ArrayList(jobs.size());
- Map keyToRecon = new HashMap();
-
- try {
- String query = buildQuery(jobs);
- String s = FreebaseUtils.mqlread(query);
-
- JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
- if (o.has("result")) {
- JSONArray results = o.getJSONArray("result");
- int count = results.length();
-
- for (int i = 0; i < count; i++) {
- JSONObject result = results.getJSONObject(i);
-
- String key = result.getJSONArray("key").getJSONObject(0).getString("value");
-
- JSONArray types = result.getJSONArray("type");
- String[] typeIDs = new String[types.length()];
- for (int j = 0; j < typeIDs.length; j++) {
- typeIDs[j] = types.getString(j);
- }
-
- ReconCandidate candidate = new ReconCandidate(
- result.getString("id"),
- result.getString("name"),
- typeIDs,
- 100
- );
-
- Recon recon = Recon.makeFreebaseRecon(historyEntryID);
- recon.addCandidate(candidate);
- recon.service = "mql";
- recon.judgment = Judgment.Matched;
- recon.judgmentAction = "auto";
- recon.match = candidate;
- recon.matchRank = 0;
-
- keyToRecon.put(key, recon);
- }
- }
- } catch (IOException e) {
- LOGGER.error("IOException during recon : ",e);
- } catch (JSONException e) {
- LOGGER.error("JSONException during recon : ",e);
- }
-
- for (ReconJob job : jobs) {
- String key = ((KeyBasedReconJob) job).key;
- Recon recon = keyToRecon.get(key);
- if (recon == null) {
- recon = createNoMatchRecon(historyEntryID);
- }
- recons.add(recon);
- }
-
- return recons;
- }
-
- private String buildQuery(List jobs)
- throws JSONException {
- String query = null;
- {
- StringWriter stringWriter = new StringWriter();
- JSONWriter jsonWriter = new JSONWriter(stringWriter);
-
- jsonWriter.array();
- jsonWriter.object();
-
- jsonWriter.key("id"); jsonWriter.value(null);
- jsonWriter.key("name"); jsonWriter.value(null);
- jsonWriter.key("guid"); jsonWriter.value(null);
- jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
-
- jsonWriter.key("key");
- jsonWriter.array();
- jsonWriter.object();
-
- jsonWriter.key("namespace");
- jsonWriter.object();
- jsonWriter.key("id"); jsonWriter.value(namespace.id);
- jsonWriter.endObject();
-
- jsonWriter.key("value"); jsonWriter.value(null);
- jsonWriter.key("value|=");
- jsonWriter.array();
- for (ReconJob job : jobs) {
- jsonWriter.value(((KeyBasedReconJob) job).key);
- }
- jsonWriter.endArray();
-
- jsonWriter.endObject();
- jsonWriter.endArray();
-
- jsonWriter.endObject();
- jsonWriter.endArray();
-
- query = stringWriter.toString();
- }
- return query;
- }
-
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
deleted file mode 100644
index 338983f1b..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.model.recon;
-
-import org.json.JSONObject;
-
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.recon.ReconConfig;
-
-abstract public class StrictReconConfig extends ReconConfig {
-
- static public ReconConfig reconstruct(JSONObject obj) throws Exception {
- String match = obj.getString("match");
- if ("key".equals(match)) {
- return KeyBasedReconConfig.reconstruct(obj);
- } else if ("id".equals(match)) {
- return IdBasedReconConfig.reconstruct(obj);
- } else if ("guid".equals(match)) {
- return GuidBasedReconConfig.reconstruct(obj);
- }
- return null;
- }
-
- @Override
- public Recon createNewRecon(long historyEntryID) {
- return Recon.makeFreebaseRecon(historyEntryID);
- }
-
- protected Recon createNoMatchRecon(long historyEntryID) {
- Recon recon = createNewRecon(historyEntryID);
- recon.service = "mql";
- recon.judgment = Judgment.None;
- recon.matchRank = -1;
- return recon;
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java
deleted file mode 100644
index d0f4b3139..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.oauth;
-
-import oauth.signpost.OAuthConsumer;
-
-import com.google.refine.freebase.util.FreebaseUtils;
-import com.google.refine.oauth.OAuthUtilities;
-import com.google.refine.oauth.Provider;
-
-public class FreebaseProvider extends Provider {
- static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"};
-
- static public void register() {
- OAuthUtilities.registerOAuthProvider(new FreebaseProvider(FreebaseUtils.FREEBASE_HOST), FREEBASE_OAUTH_INFO);
- }
-
- public FreebaseProvider(String host) {
- super(host);
- }
-
- @Override
- public String getRequestTokenServiceURL() {
- return "http://api." + host + "/api/oauth/request_token";
- }
-
- @Override
- public String getAccessTokenServiceURL() {
- return "http://api." + host + "/api/oauth/access_token";
- }
-
- @Override
- public String getUserAuthorizationURL() {
- return "http://www." + host + "/signin/app";
- }
-
- @Override
- public String getRealm() {
- return "http://api" + host + "/";
- }
-
- @Override
- public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) {
- return new FreebaseTimeCommonsHttpOAuthConsumer(consumerKey, consumerSecret);
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
deleted file mode 100644
index 54ab63afc..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.oauth;
-
-import java.io.IOException;
-
-import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.params.BasicHttpParams;
-import org.apache.http.params.HttpConnectionParams;
-import org.apache.http.params.HttpParams;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer {
-
- final static Logger logger = LoggerFactory.getLogger("oauth");
-
- private static final long serialVersionUID = -4139931605235255279L;
-
- private static final int SOCKET_TIMEOUT = 3000;
- private static final int CONNECTION_TIMEOUT = 3000;
-
- private static final String TIMER_URL = "http://refinery.freebaseapps.com/time";
-
- public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) {
- super(consumerKey, consumerSecret);
- }
-
- /**
- * It might be that the user's computer's clock is not synchronized enough with the Freebase servers
- * and this might result in Freebase thinking that it was under a replay attack.
- * To avoid this problem we get the timestamp directly from acre that we know is synchronized.
- *
- * NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same
- * could be said if we used an NTP client.
- */
- @Override
- protected String generateTimestamp() {
-
- long time = -1;
-
- try {
- HttpParams httpParams = new BasicHttpParams();
- HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT);
- HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT);
- HttpClient httpClient = new DefaultHttpClient(httpParams);
- HttpGet httpget = new HttpGet(TIMER_URL);
- HttpResponse response = httpClient.execute(httpget);
- HttpEntity entity = response.getEntity();
- if (entity != null) {
- time = Long.parseLong(EntityUtils.toString(entity),10);
- logger.debug("Got remote timestamp {}", time);
- }
- } catch (IOException e) {
- logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e);
- }
-
- if (time == -1) {
- time = System.currentTimeMillis();
- }
-
- return Long.toString(time / 1000L);
- }
-
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java
deleted file mode 100644
index e9e17afe8..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.operations;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.browsing.Engine;
-import com.google.refine.browsing.FilteredRows;
-import com.google.refine.browsing.RowVisitor;
-import com.google.refine.freebase.FreebaseType;
-import com.google.refine.freebase.model.changes.DataExtensionChange;
-import com.google.refine.freebase.util.FreebaseDataExtensionJob;
-import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
-import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
-import com.google.refine.history.HistoryEntry;
-import com.google.refine.model.AbstractOperation;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
-import com.google.refine.model.Project;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.model.Row;
-import com.google.refine.model.changes.CellAtRow;
-import com.google.refine.operations.EngineDependentOperation;
-import com.google.refine.operations.OperationRegistry;
-import com.google.refine.process.LongRunningProcess;
-import com.google.refine.process.Process;
-
-public class ExtendDataOperation extends EngineDependentOperation {
- final protected String _baseColumnName;
- final protected JSONObject _extension;
- final protected int _columnInsertIndex;
-
- static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
- JSONObject engineConfig = obj.getJSONObject("engineConfig");
-
- return new ExtendDataOperation(
- engineConfig,
- obj.getString("baseColumnName"),
- obj.getJSONObject("extension"),
- obj.getInt("columnInsertIndex")
- );
- }
-
- public ExtendDataOperation(
- JSONObject engineConfig,
- String baseColumnName,
- JSONObject extension,
- int columnInsertIndex
- ) {
- super(engineConfig);
-
- _baseColumnName = baseColumnName;
- _extension = extension;
- _columnInsertIndex = columnInsertIndex;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
- writer.key("description"); writer.value(getBriefDescription(null));
- writer.key("engineConfig"); writer.value(getEngineConfig());
- writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
- writer.key("baseColumnName"); writer.value(_baseColumnName);
- writer.key("extension"); writer.value(_extension);
- writer.endObject();
- }
-
- @Override
- protected String getBriefDescription(Project project) {
- return "Extend data at index " + _columnInsertIndex +
- " based on column " + _baseColumnName;
- }
-
- protected String createDescription(Column column, List cellsAtRows) {
- return "Extend data at index " + _columnInsertIndex +
- " based on column " + column.getName() +
- " by filling " + cellsAtRows.size();
- }
-
- @Override
- public Process createProcess(Project project, Properties options) throws Exception {
- return new ExtendDataProcess(
- project,
- getEngineConfig(),
- getBriefDescription(null)
- );
- }
-
- public class ExtendDataProcess extends LongRunningProcess implements Runnable {
- final protected Project _project;
- final protected JSONObject _engineConfig;
- final protected long _historyEntryID;
- protected int _cellIndex;
- protected FreebaseDataExtensionJob _job;
-
- public ExtendDataProcess(
- Project project,
- JSONObject engineConfig,
- String description
- ) throws JSONException {
- super(description);
- _project = project;
- _engineConfig = engineConfig;
- _historyEntryID = HistoryEntry.allocateID();
-
- _job = new FreebaseDataExtensionJob(_extension);
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("id"); writer.value(hashCode());
- writer.key("description"); writer.value(_description);
- writer.key("immediate"); writer.value(false);
- writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done"));
- writer.key("progress"); writer.value(_progress);
- writer.endObject();
- }
-
- @Override
- protected Runnable getRunnable() {
- return this;
- }
-
- protected void populateRowsWithMatches(List rowIndices) throws Exception {
- Engine engine = new Engine(_project);
- engine.initializeFromJSON(_engineConfig);
-
- Column column = _project.columnModel.getColumnByName(_baseColumnName);
- if (column == null) {
- throw new Exception("No column named " + _baseColumnName);
- }
-
- _cellIndex = column.getCellIndex();
-
- FilteredRows filteredRows = engine.getAllFilteredRows();
- filteredRows.accept(_project, new RowVisitor() {
- List _rowIndices;
-
- public RowVisitor init(List rowIndices) {
- _rowIndices = rowIndices;
- return this;
- }
-
- @Override
- public void start(Project project) {
- // nothing to do
- }
-
- @Override
- public void end(Project project) {
- // nothing to do
- }
-
- @Override
- public boolean visit(Project project, int rowIndex, Row row) {
- Cell cell = row.getCell(_cellIndex);
- if (cell != null && cell.recon != null && cell.recon.match != null) {
- _rowIndices.add(rowIndex);
- }
-
- return false;
- }
- }.init(rowIndices));
- }
-
- protected int extendRows(
- List rowIndices,
- List dataExtensions,
- int from,
- int limit,
- Map reconCandidateMap
- ) {
- Set ids = new HashSet();
-
- int end;
- for (end = from; end < limit && ids.size() < 10; end++) {
- int index = rowIndices.get(end);
- Row row = _project.rows.get(index);
- Cell cell = row.getCell(_cellIndex);
-
- ids.add(cell.recon.match.id);
- }
-
- Map map = null;
- try {
- map = _job.extend(ids, reconCandidateMap);
- } catch (Exception e) {
- map = new HashMap();
- }
-
- for (int i = from; i < end; i++) {
- int index = rowIndices.get(i);
- Row row = _project.rows.get(index);
- Cell cell = row.getCell(_cellIndex);
- String guid = cell.recon.match.id;
-
- if (map.containsKey(guid)) {
- dataExtensions.add(map.get(guid));
- } else {
- dataExtensions.add(null);
- }
- }
-
- return end;
- }
-
- @Override
- public void run() {
- List rowIndices = new ArrayList();
- List dataExtensions = new ArrayList();
-
- try {
- populateRowsWithMatches(rowIndices);
- } catch (Exception e2) {
- // TODO : Not sure what to do here?
- e2.printStackTrace();
- }
-
- int start = 0;
- Map reconCandidateMap = new HashMap();
-
- while (start < rowIndices.size()) {
- int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
- start = end;
-
- _progress = end * 100 / rowIndices.size();
- try {
- Thread.sleep(200);
- } catch (InterruptedException e) {
- if (_canceled) {
- break;
- }
- }
- }
-
- if (!_canceled) {
- List columnNames = new ArrayList();
- for (ColumnInfo info : _job.columns) {
- columnNames.add(StringUtils.join(info.names, " - "));
- }
-
- List columnTypes = new ArrayList();
- for (ColumnInfo info : _job.columns) {
- columnTypes.add(info.expectedType);
- }
-
- HistoryEntry historyEntry = new HistoryEntry(
- _historyEntryID,
- _project,
- _description,
- ExtendDataOperation.this,
- new DataExtensionChange(
- _baseColumnName,
- _columnInsertIndex,
- columnNames,
- columnTypes,
- rowIndices,
- dataExtensions,
- _historyEntryID)
- );
-
- _project.history.addEntry(historyEntry);
- _project.processManager.onDoneProcess(this);
- }
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java
deleted file mode 100644
index 10097f5c3..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.operations;
-
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.commands.UploadDataCommand;
-import com.google.refine.history.HistoryEntry;
-import com.google.refine.model.AbstractOperation;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Row;
-import com.google.refine.model.changes.MassReconChange;
-import com.google.refine.operations.OperationRegistry;
-import com.google.refine.util.ParsingUtilities;
-
-public class ImportQADataOperation extends AbstractOperation {
- static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
- return new ImportQADataOperation();
- }
-
- public ImportQADataOperation() {
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
- writer.key("description"); writer.value(getBriefDescription(null));
- writer.endObject();
- }
-
- @Override
- protected String getBriefDescription(Project project) {
- return "Import QA DAta";
- }
-
- @Override
- protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
- Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref);
- if (jobID == null) {
- throw new InternalError("Project is not associated with any data loading job.");
- }
-
- Map reconIDToResult = new HashMap();
-
- URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID);
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setReadTimeout(30000); // 30 seconds
-
- LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream()));
- try {
- String line;
- while ((line = reader.readLine()) != null) {
- JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line);
- long reconID = Long.parseLong(obj.getString("recon_id").substring(3));
-
- reconIDToResult.put(reconID, obj.getString("result"));
- }
- } finally {
- reader.close();
- }
-
- Map oldRecons = new HashMap();
- Map newRecons = new HashMap();
-
- for (int r = 0; r < project.rows.size(); r++) {
- Row row = project.rows.get(r);
-
- for (int c = 0; c < row.cells.size(); c++) {
- Cell cell = row.cells.get(c);
- if (cell != null && cell.recon != null) {
- Recon oldRecon = cell.recon;
-
- if (reconIDToResult.containsKey(oldRecon.id)) {
- Recon newRecon = oldRecon.dup();
- newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id));
-
- reconIDToResult.remove(oldRecon.id);
-
- oldRecons.put(oldRecon.id, oldRecon);
- newRecons.put(oldRecon.id, newRecon);
- }
- }
- }
- }
-
- return new HistoryEntry(
- historyEntryID,
- project,
- getBriefDescription(project),
- this,
- new MassReconChange(newRecons, oldRecons)
- );
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java
deleted file mode 100644
index b48b15e46..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.operations;
-
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.io.Writer;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.protograph.Protograph;
-import com.google.refine.history.Change;
-import com.google.refine.history.HistoryEntry;
-import com.google.refine.model.AbstractOperation;
-import com.google.refine.model.Project;
-import com.google.refine.operations.OperationRegistry;
-import com.google.refine.util.ParsingUtilities;
-import com.google.refine.util.Pool;
-
-public class SaveProtographOperation extends AbstractOperation {
- final protected Protograph _protograph;
-
- static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
- return new SaveProtographOperation(
- Protograph.reconstruct(obj.getJSONObject("protograph"))
- );
- }
-
- public SaveProtographOperation(Protograph protograph) {
- _protograph = protograph;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
- writer.key("description"); writer.value(getBriefDescription(null));
- writer.key("protograph"); _protograph.write(writer, options);
- writer.endObject();
- }
-
- @Override
- protected String getBriefDescription(Project project) {
- return "Save schema alignment skeleton";
- }
-
- @Override
- protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
- Change change = new ProtographChange(_protograph);
-
- return new HistoryEntry(historyEntryID, project, getBriefDescription(project), SaveProtographOperation.this, change);
- }
-
- static public class ProtographChange implements Change {
- final protected Protograph _newProtograph;
- protected Protograph _oldProtograph;
-
- public ProtographChange(Protograph protograph) {
- _newProtograph = protograph;
- }
-
- @Override
- public void apply(Project project) {
- synchronized (project) {
- _oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph");
-
- project.overlayModels.put("freebaseProtograph", _newProtograph);
- }
- }
-
- @Override
- public void revert(Project project) {
- synchronized (project) {
- if (_oldProtograph == null) {
- project.overlayModels.remove("freebaseProtograph");
- } else {
- project.overlayModels.put("freebaseProtograph", _oldProtograph);
- }
- }
- }
-
- @Override
- public void save(Writer writer, Properties options) throws IOException {
- writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
- writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
- writer.write("/ec/\n"); // end of change marker
- }
-
- static public Change load(LineNumberReader reader, Pool pool) throws Exception {
- Protograph oldProtograph = null;
- Protograph newProtograph = null;
-
- String line;
- while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
- int equal = line.indexOf('=');
- CharSequence field = line.subSequence(0, equal);
- String value = line.substring(equal + 1);
-
- if ("oldProtograph".equals(field) && value.length() > 0) {
- oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
- } else if ("newProtograph".equals(field) && value.length() > 0) {
- newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
- }
- }
-
- ProtographChange change = new ProtographChange(newProtograph);
- change._oldProtograph = oldProtograph;
-
- return change;
- }
-
- static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
- if (p != null) {
- JSONWriter jsonWriter = new JSONWriter(writer);
- try {
- p.write(jsonWriter, new Properties());
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java
deleted file mode 100644
index b09b32303..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseType;
-
-public class AnonymousNode implements Node, NodeWithLinks {
- final public FreebaseType type;
- final public List columnNames = new LinkedList();
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java
deleted file mode 100644
index f90ff58fb..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseType;
-
-public class CellTopicNode extends CellNode implements NodeWithLinks {
- final public FreebaseType type;
- final public List _rootNodes = new LinkedList();
-
- public int getRootNodeCount() {
- return _rootNodes.size();
- }
-
- public Node getRootNode(int index) {
- return _rootNodes.get(index);
- }
-
- @Override
- public void onBeforeSave(Project project) {
- }
-
- @Override
- public void onAfterSave(Project project) {
- }
-
-
- @Override
- public void dispose(Project project) {
- }
-
- static public Protograph reconstruct(JSONObject o) throws JSONException {
- Protograph g = new Protograph();
-
- JSONArray rootNodes = o.getJSONArray("rootNodes");
- int count = rootNodes.length();
-
- for (int i = 0; i < count; i++) {
- JSONObject o2 = rootNodes.getJSONObject(i);
- Node node = reconstructNode(o2);
- if (node != null) {
- g._rootNodes.add(node);
- }
- }
-
- return g;
- }
-
- static protected Node reconstructNode(JSONObject o) throws JSONException {
- Node node = null;
-
- String nodeType = o.getString("nodeType");
- if (nodeType.startsWith("cell-as-")) {
- if ("cell-as-topic".equals(nodeType)) {
- if (o.has("type")) {
- node = new CellTopicNode(
- reconstructType(o.getJSONObject("type"))
- );
- }
- } else if ("cell-as-value".equals(nodeType)) {
- node = new CellValueNode(
- o.getString("valueType"),
- o.getString("lang")
- );
- } else if ("cell-as-key".equals(nodeType)) {
- node = new CellKeyNode(
- reconstructTopic(o.getJSONObject("namespace"))
- );
- }
-
- if (o.has("columnName") && !o.isNull("columnName")) {
- ((CellNode) node).columnNames.add(o.getString("columnName"));
- }
- if (o.has("columnNames") && !o.isNull("columnNames")) {
- JSONArray columnNames = o.getJSONArray("columnNames");
- int count = columnNames.length();
-
- for (int c = 0; c < count; c++) {
- ((CellNode) node).columnNames.add(columnNames.getString(c));
- }
- }
- } else if ("topic".equals(nodeType)) {
- node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
- } else if ("value".equals(nodeType)) {
- node = new ValueNode(
- o.get("value"),
- o.getString("valueType"),
- o.getString("lang")
- );
- } else if ("anonymous".equals(nodeType)) {
- node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
- }
-
- if (node != null && node instanceof NodeWithLinks && o.has("links")) {
- NodeWithLinks node2 = (NodeWithLinks) node;
-
- JSONArray links = o.getJSONArray("links");
- int linkCount = links.length();
-
- for (int j = 0; j < linkCount; j++) {
- JSONObject oLink = links.getJSONObject(j);
- Condition condition = null;
-
- if (oLink.has("condition") && !oLink.isNull("condition")) {
- JSONObject oCondition = oLink.getJSONObject("condition");
- if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
- condition = new BooleanColumnCondition(oCondition.getString("columnName"));
- }
- }
-
- node2.addLink(new Link(
- reconstructProperty(oLink.getJSONObject("property")),
- oLink.has("target") && !oLink.isNull("target") ?
- reconstructNode(oLink.getJSONObject("target")) : null,
- condition,
- oLink.has("load") && !oLink.isNull("load") ?
- oLink.getBoolean("load") : true
- ));
- }
- }
-
- return node;
- }
-
- static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
- return new FreebaseProperty(
- o.getString("id"),
- o.getString("name")
- );
- }
-
- static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
- return new FreebaseType(
- o.getString("id"),
- o.getString("name")
- );
- }
-
- static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
- return new FreebaseTopic(
- o.getString("id"),
- o.getString("name")
- );
- }
-
- @Override
- public void write(JSONWriter writer, Properties options) throws JSONException {
- writer.object();
- writer.key("rootNodes"); writer.array();
-
- for (Node node : _rootNodes) {
- node.write(writer, options);
- }
-
- writer.endArray();
- writer.endObject();
- }
-
- static public Protograph load(Project project, JSONObject obj) throws Exception {
- return reconstruct(obj);
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java
deleted file mode 100644
index d8befdafa..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph;
-
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-public class ValueNode implements Node {
- final public Object value;
- final public String valueType;
- final public String lang;
-
- public ValueNode(Object value, String valueType, String lang) {
- this.value = value;
- this.valueType = valueType;
- this.lang = lang;
- }
-
- @Override
- public void write(JSONWriter writer, Properties options)
- throws JSONException {
-
- writer.object();
- writer.key("nodeType"); writer.value("value");
- writer.key("value"); writer.value(value);
- writer.key("valueType"); writer.value(valueType);
- writer.key("lang"); writer.value(lang);
- writer.endObject();
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java
deleted file mode 100644
index 7a284e0a2..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph.transpose;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseProperty;
-import com.google.refine.freebase.protograph.AnonymousNode;
-import com.google.refine.freebase.protograph.CellKeyNode;
-import com.google.refine.freebase.protograph.CellNode;
-import com.google.refine.freebase.protograph.CellTopicNode;
-import com.google.refine.freebase.protograph.CellValueNode;
-import com.google.refine.freebase.protograph.FreebaseTopicNode;
-import com.google.refine.freebase.protograph.Link;
-import com.google.refine.freebase.protograph.ValueNode;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Recon;
-import com.google.refine.util.JSONUtilities;
-
-public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
- protected Writer writer;
- protected List rootObjects = new LinkedList();
-
- private static final String TYPE = "type";
- private static final String ID = "id";
- private static final String NAME = "name";
- private static final String CREATE = "create";
- private static final String VALUE = "value";
- private static final String CONNECT = "connect";
- private static final String LANG = "lang";
-
- public MqlwriteLikeTransposedNodeFactory(Writer writer) {
- this.writer = writer;
- }
-
- protected JSONArray getJSON() {
- return new JSONArray(rootObjects);
- }
-
- @Override
- public void flush() throws IOException {
- try {
- JSONWriter jsonWriter = new JSONWriter(writer);
-
- jsonWriter.array();
- for (JSONObject obj : rootObjects) {
- jsonWriter.value(obj);
- }
- jsonWriter.endArray();
-
- } catch (JSONException e) {
- e.printStackTrace();
- }
- writer.flush();
- }
-
- abstract protected class JsonTransposedNode implements TransposedNode {
- abstract public Object getJSON();
- }
-
- abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
- abstract public JSONObject getJSONObject();
-
- protected JSONObject obj;
-
- @Override
- public Object getJSON() {
- return getJSONObject();
- }
- }
-
- protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
- JsonObjectTransposedNode parent;
- FreebaseProperty property;
- AnonymousNode node;
-
- protected AnonymousTransposedNode(
- JsonObjectTransposedNode parent,
- FreebaseProperty property,
- AnonymousNode node
- ) {
- this.parent = parent;
- this.property = property;
- this.node = node;
- }
-
- @Override
- public JSONObject getJSONObject() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- obj.put(TYPE, this.node.type.id);
- obj.put(ID, (String) null);
- obj.put(CREATE, "unconditional");
- } catch (JSONException e) {
- e.printStackTrace();
- }
-
- linkTransposedNodeJSON(obj, parent, property);
- }
-
- return obj;
- }
- }
-
- protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
- protected CellTopicNode node;
- protected Cell cell;
-
- public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
- this.node = node;
- this.cell = cell;
- }
-
- @Override
- public JSONObject getJSONObject() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- if (cell.recon != null &&
- cell.recon.judgment == Recon.Judgment.Matched &&
- cell.recon.match != null) {
- obj.put(ID, cell.recon.match.id);
- } else {
- obj.put(ID, (String) null);
- obj.put(NAME, cell.value.toString());
- obj.put(TYPE, node.type.id);
- obj.put(CREATE, "unless_exists");
- }
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- return obj;
- }
- }
-
- protected class CellValueTransposedNode extends JsonTransposedNode {
- protected JSONObject obj;
- protected CellValueNode node;
- protected Cell cell;
-
- public CellValueTransposedNode(CellValueNode node, Cell cell) {
- this.node = node;
- this.cell = cell;
- }
-
- @Override
- public Object getJSON() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- JSONUtilities.putField(obj, VALUE, cell.value);
-
- obj.put(TYPE, node.valueType);
- if ("/type/text".equals(node.valueType)) {
- obj.put(LANG, node.lang);
- }
-
- obj.put(CONNECT, "insert");
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- return obj;
- }
- }
-
- protected class CellKeyTransposedNode extends JsonTransposedNode {
- protected JSONObject obj;
- protected CellKeyNode node;
- protected Cell cell;
-
- public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
- this.node = node;
- this.cell = cell;
- }
-
- @Override
- public Object getJSON() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- obj.put(VALUE, cell.value.toString());
-
- JSONObject nsObj = new JSONObject();
- nsObj.put(ID, node.namespace.id);
-
- obj.put("namespace", nsObj);
- obj.put(CONNECT, "insert");
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- return obj;
- }
- }
-
- protected class TopicTransposedNode extends JsonObjectTransposedNode {
- protected FreebaseTopicNode node;
-
- public TopicTransposedNode(FreebaseTopicNode node) {
- this.node = node;
- }
-
- @Override
- public JSONObject getJSONObject() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- obj.put(ID, node.topic.id);
- // TODO: This won't work at the root of the query, so that needs
- // to be special cased, but for now one must use a different shaped graph
- // (ie move the Freebase topic to someplace other than the root
- obj.put(CONNECT, "insert");
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- return obj;
- }
- }
-
- protected class ValueTransposedNode extends JsonTransposedNode {
- protected JSONObject obj;
- protected ValueNode node;
-
- public ValueTransposedNode(ValueNode node) {
- this.node = node;
- }
-
- @Override
- public Object getJSON() {
- if (obj == null) {
- obj = new JSONObject();
- try {
- if ("/type/datetime".equals(node.valueType) && node.value instanceof Long) {
- // Special case integers as year-only dates
- obj.put(VALUE, node.value.toString());
- } else {
- obj.put(VALUE, node.value);
- }
- obj.put(TYPE, node.valueType);
- if ("/type/text".equals(node.valueType)) {
- obj.put(LANG, node.lang);
- }
-
- obj.put(CONNECT, "insert");
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- return obj;
- }
- }
- @Override
- public TransposedNode transposeAnonymousNode(
- TransposedNode parentNode,
- Link link,
- AnonymousNode node, int rowIndex) {
-
- return new AnonymousTransposedNode(
- parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
- link != null ? link.property : null,
- node
- );
- }
-
- @Override
- public TransposedNode transposeCellNode(
- TransposedNode parentNode,
- Link link,
- CellNode node,
- int rowIndex,
- int cellIndex,
- Cell cell) {
-
- JsonTransposedNode tnode = null;
- if (node instanceof CellTopicNode) {
- tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
- } else if (node instanceof CellValueNode) {
- tnode = new CellValueTransposedNode((CellValueNode) node, cell);
- } else if (node instanceof CellKeyNode) {
- tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
- }
-
- if (tnode != null) {
- processTransposedNode(tnode, parentNode, link != null ? link.property : null);
- }
- return tnode;
- }
-
- @Override
- public TransposedNode transposeTopicNode(
- TransposedNode parentNode,
- Link link,
- FreebaseTopicNode node, int rowIndex) {
-
- JsonTransposedNode tnode = new TopicTransposedNode(node);
-
- processTransposedNode(tnode, parentNode, link != null ? link.property : null);
-
- return tnode;
- }
-
- @Override
- public TransposedNode transposeValueNode(
- TransposedNode parentNode,
- Link link,
- ValueNode node, int rowIndex) {
-
- JsonTransposedNode tnode = new ValueTransposedNode(node);
-
- processTransposedNode(tnode, parentNode, link != null ? link.property : null);
-
- return tnode;
- }
-
- protected void processTransposedNode(
- JsonTransposedNode tnode,
- TransposedNode parentNode,
- FreebaseProperty property
- ) {
-
- if (!(tnode instanceof AnonymousTransposedNode)) {
- linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
- }
- }
-
- protected void linkTransposedNodeJSON(
- Object obj,
- TransposedNode parentNode,
- FreebaseProperty property
- ) {
-
- if (parentNode == null) {
- if (obj instanceof JSONObject) {
- rootObjects.add((JSONObject) obj);
- }
- } else if (parentNode instanceof JsonTransposedNode) {
- JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
-
- try {
- JSONArray a = null;
- if (parentObj.has(property.id)) {
- a = parentObj.getJSONArray(property.id);
- } else {
- a = new JSONArray();
- parentObj.put(property.id, a);
- }
-
- a.put(a.length(), obj);
- } catch (JSONException e) {
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/SchemaHelper.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/SchemaHelper.java
deleted file mode 100644
index fdfdea700..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/SchemaHelper.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package com.google.refine.freebase.protograph.transpose;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import com.google.refine.util.JSONUtilities;
-import com.google.refine.util.ParsingUtilities;
-
-public class SchemaHelper {
- static private final String helperURL = "http://openrefine-helper.freebaseapps.com/";
-
- static private class PropertyInfo {
- String fromTypeID;
- String toTypeID;
- }
-
- static private class TypeInfo {
- String[] includedTypes;
- }
-
- protected Map properties = new HashMap();
- protected Map types = new HashMap();
-
- public String getPropertyFromType(String propertyID) {
- ensureProperty(propertyID);
- return properties.get(propertyID).fromTypeID;
- }
- public String getPropertyToType(String propertyID) {
- ensureProperty(propertyID);
- return properties.get(propertyID).toTypeID;
- }
- public String[] getIncludedTypeIDs(String typeID) {
- ensureType(typeID);
- return types.get(typeID).includedTypes;
- }
-
- private void ensureProperty(String propertyID) {
- if (properties.containsKey(propertyID)) {
- return;
- }
-
- PropertyInfo info = new PropertyInfo();
- properties.put(propertyID, info);
-
- JSONObject obj = getJson(helperURL + "get_property_data" + propertyID);
- if (obj != null) {
- try {
- if (!obj.isNull("from")) {
- info.fromTypeID = obj.getString("from");
- }
- } catch (JSONException e) {
- }
- try {
- if (!obj.isNull("to")) {
- info.toTypeID = obj.getString("to");
- }
- } catch (JSONException e) {
- }
- }
- }
-
- private void ensureType(String typeID) {
- if (types.containsKey(typeID)) {
- return;
- }
-
- TypeInfo info = new TypeInfo();
- types.put(typeID, info);
-
- JSONObject obj = getJson(helperURL + "get_type_data" + typeID);
- if (obj != null) {
- if (!obj.isNull("includes")) {
- info.includedTypes = JSONUtilities.getStringArray(obj, "includes");
- }
- }
- }
-
- private JSONObject getJson(String urlString) {
- try {
- URL url = new URL(urlString);
- InputStream is = url.openStream();
- try {
- String s = ParsingUtilities.inputStreamToString(is);
- return ParsingUtilities.evaluateJsonStringToObject(s);
- } finally {
- is.close();
- }
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (JSONException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return null;
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java
deleted file mode 100644
index 7b97f70e9..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph.transpose;
-
-public interface TransposedNode {
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java
deleted file mode 100644
index b61afa1d6..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph.transpose;
-
-import java.io.IOException;
-
-import com.google.refine.freebase.protograph.AnonymousNode;
-import com.google.refine.freebase.protograph.CellNode;
-import com.google.refine.freebase.protograph.FreebaseTopicNode;
-import com.google.refine.freebase.protograph.Link;
-import com.google.refine.freebase.protograph.ValueNode;
-import com.google.refine.model.Cell;
-
-public interface TransposedNodeFactory {
- public TransposedNode transposeAnonymousNode(
- TransposedNode parentNode,
- Link link,
- AnonymousNode node, int rowIndex
- );
-
- public TransposedNode transposeCellNode(
- TransposedNode parentNode,
- Link link,
- CellNode node,
- int rowIndex,
- int cellIndex,
- Cell cell
- );
-
- public TransposedNode transposeValueNode(
- TransposedNode parentNode,
- Link link,
- ValueNode node,
- int rowIndex
- );
-
- public TransposedNode transposeTopicNode(
- TransposedNode parentNode,
- Link link,
- FreebaseTopicNode node,
- int rowIndex
- );
-
- public void flush() throws IOException;
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java
deleted file mode 100644
index 4e65cab52..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
-
-Copyright 2010, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph.transpose;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import com.google.refine.browsing.FilteredRows;
-import com.google.refine.browsing.RowVisitor;
-import com.google.refine.expr.ExpressionUtils;
-import com.google.refine.freebase.protograph.AnonymousNode;
-import com.google.refine.freebase.protograph.CellNode;
-import com.google.refine.freebase.protograph.CellTopicNode;
-import com.google.refine.freebase.protograph.FreebaseTopicNode;
-import com.google.refine.freebase.protograph.Link;
-import com.google.refine.freebase.protograph.Node;
-import com.google.refine.freebase.protograph.NodeWithLinks;
-import com.google.refine.freebase.protograph.Protograph;
-import com.google.refine.freebase.protograph.ValueNode;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.Row;
-
-public class Transposer {
- static public void transpose(
- Project project,
- FilteredRows filteredRows,
- Protograph protograph,
- Node rootNode,
- TransposedNodeFactory nodeFactory
- ) {
- transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
- }
-
- static public void transpose(
- Project project,
- FilteredRows filteredRows,
- Protograph protograph,
- Node rootNode,
- TransposedNodeFactory nodeFactory,
- int limit
- ) {
- Context rootContext = new Context(rootNode, null, null, limit);
-
- filteredRows.accept(project, new RowVisitor() {
- Context rootContext;
- Protograph protograph;
- Node rootNode;
- TransposedNodeFactory nodeFactory;
-
- @Override
- public boolean visit(Project project, int rowIndex, Row row) {
- if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
- descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
- }
-
- if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
- return true;
- }
- return false;
- }
-
- @Override
- public void start(Project project) {
- }
-
- @Override
- public void end(Project project) {
- }
-
- public RowVisitor init(
- Context rootContext,
- Protograph protograph,
- Node rootNode,
- TransposedNodeFactory nodeFactory
- ) {
- this.rootContext = rootContext;
- this.protograph = protograph;
- this.rootNode = rootNode;
- this.nodeFactory = nodeFactory;
-
- return this;
- }
- }.init(rootContext, protograph, rootNode, nodeFactory));
- }
-
- static protected void descend(
- Project project,
- Protograph protograph,
- TransposedNodeFactory nodeFactory,
- int rowIndex,
- Row row,
- Node node,
- Context context
- ) {
- List tnodes = new LinkedList();
-
- Link link = context.parent == null ? null : context.link;
-
- if (node instanceof CellNode) {
- if (!descendCellNode(project, nodeFactory, rowIndex, row, node, context, tnodes, link)) {
- return;
- }
- } else if (node instanceof AnonymousNode) {
- descendAnonymousNode(nodeFactory, rowIndex, node, context, tnodes, link);
- } else if (node instanceof FreebaseTopicNode) {
- descendFreebaseTopicNode(nodeFactory, rowIndex, node, context, tnodes, link);
- } else if (node instanceof ValueNode) {
- descendValueNode(nodeFactory, rowIndex, node, context, tnodes, link);
- }
-
- if (tnodes.size() > 0) {
- context.transposedNodes.clear();
- context.transposedNodes.addAll(tnodes);
- }
-
- if (node instanceof NodeWithLinks) {
- NodeWithLinks node2 = (NodeWithLinks) node;
- int linkCount = node2.getLinkCount();
-
- for (int i = 0; i < linkCount; i++) {
- Link link2 = node2.getLink(i);
- if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
- descend(
- project,
- protograph,
- nodeFactory,
- rowIndex,
- row,
- link2.getTarget(),
- context.subContexts.get(i)
- );
- }
- }
- }
- }
-
- private static boolean descendCellNode(Project project, TransposedNodeFactory nodeFactory, int rowIndex, Row row,
- Node node, Context context, List tnodes, Link link) {
- CellNode node2 = (CellNode) node;
- for (String columnName : node2.columnNames) {
- Column column = project.columnModel.getColumnByName(columnName);
- if (column != null) {
- int cellIndex = column.getCellIndex();
-
- Cell cell = row.getCell(cellIndex);
- if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
- if (node2 instanceof CellTopicNode &&
- (cell.recon == null || cell.recon.judgment == Judgment.None)) {
- return false;
- }
-
- context.count++;
- if (context.limit > 0 && context.count > context.limit) {
- return false;
- }
-
- if (context.parent == null) {
- tnodes.add(nodeFactory.transposeCellNode(
- null,
- link,
- node2,
- rowIndex,
- cellIndex,
- cell
- ));
- } else {
- for (TransposedNode parentNode : context.parent.transposedNodes) {
- tnodes.add(nodeFactory.transposeCellNode(
- parentNode,
- link,
- node2,
- rowIndex,
- cellIndex,
- cell
- ));
- }
- }
- }
- }
- }
- return true;
- }
-
- private static void descendAnonymousNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node,
- Context context, List tnodes, Link link) {
- if (context.parent == null) {
- tnodes.add(nodeFactory.transposeAnonymousNode(
- null,
- link,
- (AnonymousNode) node,
- rowIndex
- ));
- } else {
- for (TransposedNode parentNode : context.parent.transposedNodes) {
- tnodes.add(nodeFactory.transposeAnonymousNode(
- parentNode,
- link,
- (AnonymousNode) node,
- rowIndex
- ));
- }
- }
- }
-
- private static void descendFreebaseTopicNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node,
- Context context, List tnodes, Link link) {
- if (context.parent == null) {
- tnodes.add(nodeFactory.transposeTopicNode(
- null,
- link,
- (FreebaseTopicNode) node,
- rowIndex
- ));
- } else {
- for (TransposedNode parentNode : context.parent.transposedNodes) {
- tnodes.add(nodeFactory.transposeTopicNode(
- parentNode,
- link,
- (FreebaseTopicNode) node,
- rowIndex
- ));
- }
- }
- }
-
- private static void descendValueNode(TransposedNodeFactory nodeFactory, int rowIndex, Node node, Context context,
- List tnodes, Link link) {
- if (context.parent == null) {
- tnodes.add(nodeFactory.transposeValueNode(
- null,
- link,
- (ValueNode) node,
- rowIndex
- ));
- } else {
- for (TransposedNode parentNode : context.parent.transposedNodes) {
- tnodes.add(nodeFactory.transposeValueNode(
- parentNode,
- link,
- (ValueNode) node,
- rowIndex
- ));
- }
- }
- }
-
- static class Context {
- List transposedNodes = new LinkedList();
- List subContexts;
- Context parent;
- Link link;
- int count;
- int limit;
-
- Context(Node node, Context parent, Link link, int limit) {
- this.parent = parent;
- this.link = link;
- this.limit = limit;
-
- if (node instanceof NodeWithLinks) {
- NodeWithLinks node2 = (NodeWithLinks) node;
-
- int subContextCount = node2.getLinkCount();
-
- subContexts = new LinkedList();
- for (int i = 0; i < subContextCount; i++) {
- Link link2 = node2.getLink(i);
- subContexts.add(
- new Context(link2.getTarget(), this, link2, -1));
- }
- }
- }
-
- public void nullifySubContextNodes() {
- if (subContexts != null) {
- for (Context context : subContexts) {
- context.transposedNodes.clear();
- context.nullifySubContextNodes();
- }
- }
- }
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java
deleted file mode 100644
index f2a88074a..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java
+++ /dev/null
@@ -1,899 +0,0 @@
-/*
-
-Copyright 2010,2012. Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.protograph.transpose;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import com.google.refine.freebase.FreebaseProperty;
-import com.google.refine.freebase.FreebaseTopic;
-import com.google.refine.freebase.protograph.AnonymousNode;
-import com.google.refine.freebase.protograph.CellKeyNode;
-import com.google.refine.freebase.protograph.CellNode;
-import com.google.refine.freebase.protograph.CellTopicNode;
-import com.google.refine.freebase.protograph.CellValueNode;
-import com.google.refine.freebase.protograph.FreebaseTopicNode;
-import com.google.refine.freebase.protograph.Link;
-import com.google.refine.freebase.protograph.ValueNode;
-import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
-import com.google.refine.model.Project;
-import com.google.refine.model.Recon;
-import com.google.refine.model.Recon.Judgment;
-import com.google.refine.model.recon.ReconConfig;
-import com.google.refine.model.recon.StandardReconConfig;
-
-public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
- protected Project project;
-
- protected boolean start = true;
- protected Writer writer;
- protected WritingTransposedNode lastRootNode;
- protected Map varPool = new HashMap();
- protected Map newTopicVars = new HashMap();
- protected Set serializedRecons = new HashSet();
-
- protected long contextID = 0;
- protected int contextRowIndex;
- protected int contextRefCount = 0;
- protected JSONObject contextTreeRoot;
-
- protected SchemaHelper schemaHelper = new SchemaHelper();
-
- protected Map> typeIDToAssertedReconIDs = new HashMap>();
- protected Set getAssertedReconIDSet(String typeID) {
- Set assertedReconIDSet = typeIDToAssertedReconIDs.get(typeID);
- if (assertedReconIDSet == null) {
- assertedReconIDSet = new HashSet();
- typeIDToAssertedReconIDs.put(typeID, assertedReconIDSet);
- }
- return assertedReconIDSet;
- }
- protected void ensureOneTypeAsserted(Recon recon, String typeID) {
- Set assertedReconIDSet = getAssertedReconIDSet(typeID);
- if (!assertedReconIDSet.contains(recon.id)) {
- assertedReconIDSet.add(recon.id);
-
- String subject = recon.judgment == Judgment.New ? newTopicVars.get(recon.id) : recon.match.id;
-
- StringBuffer sb = new StringBuffer();
- sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
- sb.append(", \"p\" : \"type\"");
- sb.append(", \"o\" : \""); sb.append(typeID); sb.append('"');
- sb.append(" }");
-
- writeLine(sb.toString());
- }
- }
- protected void ensureAllIncludedTypesAsserted(Recon recon, String typeID) {
- ensureOneTypeAsserted(recon, typeID);
-
- String[] includedTypeIDs = schemaHelper.getIncludedTypeIDs(typeID);
- if (includedTypeIDs != null) {
- for (String typeID2 : includedTypeIDs) {
- if (!"/type/object".equals(typeID2)) {
- ensureOneTypeAsserted(recon, typeID2);
- }
- }
- }
- }
- protected void ensureFromTypesAsserted(Recon recon, String propertyID) {
- String fromTypeID = schemaHelper.getPropertyFromType(propertyID);
- if (fromTypeID != null && !"/type/object".equals(fromTypeID)) {
- ensureAllIncludedTypesAsserted(recon, fromTypeID);
- }
- }
- protected void ensureToTypesAsserted(Recon recon, String propertyID) {
- String toTypeID = schemaHelper.getPropertyToType(propertyID);
- if (toTypeID != null && !"/type/object".equals(toTypeID)) {
- ensureAllIncludedTypesAsserted(recon, toTypeID);
- }
- }
-
- public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
- this.project = project;
- this.writer = writer;
- }
-
- @Override
- public void flush() throws IOException {
- if (lastRootNode != null) {
- lastRootNode.write(null, null, project, -1, -1, null);
- lastRootNode = null;
-
- writeContextTreeNode();
- }
- }
-
- protected void writeLine(String line) {
- try {
- if (start) {
- start = false;
- } else {
- writer.write('\n');
- }
- writer.write(line);
- } catch (IOException e) {
- // ignore
- }
- }
-
- protected void writeRecon(
- StringBuffer sb,
- Project project,
- int rowIndex,
- int cellIndex,
- Cell cell
- ) {
- Recon recon = cell.recon;
-
- sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
- contextRefCount++;
-
- if (!serializedRecons.contains(recon.id)) {
- serializedRecons.add(recon.id);
-
- Column column = project.columnModel.getColumnByCellIndex(cellIndex);
-
- // qa:sample_group
- {
- StringBuffer sb2 = new StringBuffer();
-
- sb2.append("{ \"s\" : \"rec");
- sb2.append(Long.toString(recon.id));
- sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
- sb2.append(JSONObject.quote(column.getName()));
- sb2.append(", \"ignore\" : true }");
-
- writeLine(sb2.toString());
- }
-
- // qa:recon_data
- {
- StringBuffer sb2 = new StringBuffer();
-
- String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
-
- sb2.append("{ \"s\" : \"rec");
- sb2.append(Long.toString(recon.id));
- sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
-
- sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
- sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
- sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
- sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
- sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
- sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
-
- if (recon.judgment == Judgment.Matched) {
- sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
- sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
- }
-
- ReconConfig reconConfig = column.getReconConfig();
- if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
- StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
- sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
- }
-
- sb2.append(" } }");
-
- writeLine(sb2.toString());
- }
- }
- }
-
- protected void writeLine(
- String subject, String predicate, Object object,
- Project project,
- int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
- int objectRowIndex, int objectCellIndex, Cell objectCell,
- boolean ignore
- ) {
- if (subject != null && object != null) {
- String s = object instanceof String ?
- JSONObject.quote((String) object) : object.toString();
-
- StringBuffer sb = new StringBuffer();
- sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
- sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
- sb.append(", \"o\" : "); sb.append(s);
- if (subjectCell != null || objectCell != null) {
- sb.append(", \"meta\" : { ");
-
- sb.append("\"recon\" : { ");
- if (subjectCell != null) {
- sb.append("\"s\" : ");
- writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
- }
- if (objectCell != null) {
- if (subjectCell != null) {
- sb.append(", ");
- }
- sb.append("\"o\" : ");
- writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
- }
- sb.append(" }");
-
- sb.append(" }");
- }
- if (ignore) {
- sb.append(", \"ignore\" : true");
- }
- sb.append(" }");
-
- writeLine(sb.toString());
- }
- }
-
- protected void writeLine(
- String subject, String predicate, Object object, String lang,
- Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
- boolean ignore
- ) {
- if (subject != null && object != null) {
- String s = object instanceof String ?
- JSONObject.quote((String) object) : object.toString();
-
- StringBuffer sb = new StringBuffer();
- sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
- sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
- sb.append(", \"o\" : "); sb.append(s);
- sb.append(", \"lang\" : \""); sb.append(lang); sb.append('"');
-
- if (subjectCell != null) {
- sb.append(", \"meta\" : { ");
- sb.append("\"recon\" : { ");
- sb.append("\"s\" : ");
- writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
- sb.append(" }");
- sb.append(" }");
- }
- if (ignore) {
- sb.append(", \"ignore\" : true");
- }
- sb.append(" }");
-
- writeLine(sb.toString());
- }
- }
-
- abstract protected class WritingTransposedNode implements TransposedNode {
- JSONObject jsonContextNode;
- boolean load;
-
- public Object write(
- String subject, String predicate, Project project,
- int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
-
- return internalWrite(
- subject, predicate, project,
- subjectRowIndex, subjectCellIndex, subjectCell);
- }
-
- abstract public Object internalWrite(
- String subject, String predicate, Project project,
- int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
- }
-
- abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
- public List rowIndices = new LinkedList();
- public List children = new LinkedList();
-
- protected void writeChildren(
- String subject, Project project,
- int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
-
- Recon recon = subjectCell != null && subjectCell.recon != null &&
- (subjectCell.recon.judgment == Judgment.Matched || subjectCell.recon.judgment == Judgment.New)
- ? subjectCell.recon : null;
-
- for (int i = 0; i < children.size(); i++) {
- WritingTransposedNode child = children.get(i);
- Link link = links.get(i);
- String predicate = link.property.id;
-
- if (recon != null) {
- ensureFromTypesAsserted(recon, predicate);
- }
-
- child.write(subject, predicate, project,
- subjectRowIndex, subjectCellIndex, subjectCell);
- }
- }
- }
-
- protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
-
- //protected AnonymousTransposedNode(AnonymousNode node) { }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- if (children.size() == 0 || subject == null) {
- return null;
- }
-
- StringBuffer sb = new StringBuffer();
- sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
- sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
- sb.append(", \"o\" : { ");
-
- StringBuffer sbRecon = new StringBuffer();
-
- boolean first = true;
- boolean firstRecon = true;
-
- if (subjectCell != null && subjectCell.recon != null) {
- sbRecon.append("\"s\" : ");
- writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
-
- firstRecon = false;
- }
-
- for (int i = 0; i < children.size(); i++) {
- WritingTransposedNode child = children.get(i);
- Link link = links.get(i);
-
- FreebaseProperty property = link.property;
-
- Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
- if (c != null) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
- }
- sb.append("\"" + property.id + "\": ");
- sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
- }
-
- if (child instanceof CellTopicTransposedNode) {
- CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
- Recon recon = child2.cell.recon;
-
- if (recon != null &&
- (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
-
- if (firstRecon) {
- firstRecon = false;
- } else {
- sbRecon.append(", ");
- }
-
- sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
-
- writeRecon(sbRecon, project,
- rowIndices.get(i), child2.cellIndex, child2.cell);
- }
- }
- }
- sb.append(" }, \"meta\" : { \"recon\" : { ");
- sb.append(sbRecon.toString());
- sb.append(" } } }");
-
- writeLine(sb.toString());
-
- return null;
- }
- }
-
- protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
- protected CellTopicNode node;
- protected int rowIndex;
- protected int cellIndex;
- protected Cell cell;
-
- public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
- this.node = node;
- this.rowIndex = rowIndex;
- this.cellIndex = cellIndex;
- this.cell = cell;
- }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- String id = null;
- if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
- int objectRowIndex = rowIndex;
- int objectCellIndex = cellIndex;
- Cell objectCell = cell;
-
- String typeID = node.type.id;
-
- Column column = project.columnModel.getColumnByCellIndex(cellIndex);
- ReconConfig reconConfig = column.getReconConfig();
- if (reconConfig instanceof StandardReconConfig) {
- typeID = ((StandardReconConfig) reconConfig).typeID;
- }
-
- if (cell.recon.judgment == Recon.Judgment.Matched) {
- id = cell.recon.match.id;
-
- } else if (cell.recon.judgment == Judgment.New) {
- if (newTopicVars.containsKey(cell.recon.id)) {
- id = newTopicVars.get(cell.recon.id);
- } else {
- String columnName = column.getName();
-
- long var = 0;
- if (varPool.containsKey(columnName)) {
- var = varPool.get(columnName);
- }
- varPool.put(columnName, var + 1);
-
- id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
-
- writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
- writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
-
- getAssertedReconIDSet(typeID).add(cell.recon.id);
-
- newTopicVars.put(cell.recon.id, id);
- }
- } else {
- return null;
- }
-
- ensureAllIncludedTypesAsserted(cell.recon, typeID);
-
- if (subject != null) {
- ensureToTypesAsserted(cell.recon, predicate);
-
- writeLine(subject, predicate, id, project,
- subjectRowIndex, subjectCellIndex, subjectCell,
- objectRowIndex, objectCellIndex, objectCell, !load);
- }
-
- writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
- }
-
- return id;
- }
- }
-
- protected class CellValueTransposedNode extends WritingTransposedNode {
- protected JSONObject obj;
- protected CellValueNode node;
- protected int rowIndex;
- protected int cellIndex;
- protected Cell cell;
-
- public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
- this.node = node;
- this.rowIndex = rowIndex;
- this.cellIndex = cellIndex;
- this.cell = cell;
- }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- Object value = cell.value;
- if (value != null) {
- if ("/type/text".equals(node.valueType)) {
- value = value.toString();
- if (subject != null) {
- writeLine(subject, predicate, value, node.lang, project,
- subjectRowIndex, subjectCellIndex, subjectCell, !load);
- }
- } else {
- value = validateValue(value, node.valueType);
- if (subject != null && value != null) {
- writeLine(subject, predicate, value, project,
- subjectRowIndex, subjectCellIndex, subjectCell,
- -1, -1, null, !load);
- }
- }
- }
-
- return value;
- }
- }
-
- protected class CellKeyTransposedNode extends WritingTransposedNode {
- protected CellKeyNode node;
- protected int rowIndex;
- protected int cellIndex;
- protected Cell cell;
-
- public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
- this.node = node;
- this.rowIndex = rowIndex;
- this.cellIndex = cellIndex;
- this.cell = cell;
- }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
- subjectRowIndex, subjectCellIndex, subjectCell,
- -1, -1, null, !load);
-
- return null;
- }
- }
-
- protected class TopicTransposedNode extends TransposedNodeWithChildren {
- protected FreebaseTopicNode node;
-
- public TopicTransposedNode(FreebaseTopicNode node) {
- this.node = node;
- }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- writeLine(subject, predicate, node.topic.id, project,
- subjectRowIndex, subjectCellIndex, subjectCell,
- -1, -1, null, !load);
-
- writeChildren(node.topic.id, project, -1, -1, null);
-
- return node.topic.id;
- }
- }
-
- protected class ValueTransposedNode extends WritingTransposedNode {
- protected ValueNode node;
-
- public ValueTransposedNode(ValueNode node) {
- this.node = node;
- }
-
- @Override
- public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
- Object value = node.value;
- if (value != null) {
- if ("/type/text".equals(node.valueType)) {
- value = value.toString();
- if (subject != null) {
- writeLine(subject, predicate, value, node.lang, project,
- subjectRowIndex, subjectCellIndex, subjectCell, !load);
- }
- } else {
- value = validateValue(value, node.valueType);
- if (subject != null && value != null) {
- writeLine(subject, predicate, value, project,
- subjectRowIndex, subjectCellIndex, subjectCell,
- -1, -1, null, !load);
- }
- }
- }
- return value;
- }
- }
-
- @Override
- public TransposedNode transposeAnonymousNode(
- TransposedNode parentNode,
- Link link,
- AnonymousNode node, int rowIndex) {
-
- WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
- WritingTransposedNode tnode = new AnonymousTransposedNode();
-
- tnode.load =
- (parentNode2 == null || parentNode2.load) &&
- (link == null || link.load);
-
- processTransposedNode(tnode, parentNode, link, rowIndex);
-
- tnode.jsonContextNode = addJsonContext(
- parentNode2 != null ? parentNode2.jsonContextNode : null,
- link != null ? link.property.id : null,
- null
- );
-
- return tnode;
- }
-
- @Override
- public TransposedNode transposeCellNode(
- TransposedNode parentNode,
- Link link,
- CellNode node,
- int rowIndex,
- int cellIndex,
- Cell cell) {
-
- WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
-
- WritingTransposedNode tnode = null;
- if (node instanceof CellTopicNode) {
- if (cell.recon != null &&
- (cell.recon.judgment == Judgment.Matched ||
- cell.recon.judgment == Judgment.New)) {
-
- tnode = new CellTopicTransposedNode(
- (CellTopicNode) node, rowIndex, cellIndex, cell);
- }
- } else if (node instanceof CellValueNode) {
- tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
- } else if (node instanceof CellKeyNode) {
- tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
- }
-
- if (tnode != null) {
- tnode.load =
- (parentNode2 == null || parentNode2.load) &&
- (link == null || link.load);
-
- processTransposedNode(tnode, parentNode, link, rowIndex);
-
- tnode.jsonContextNode = addJsonContext(
- parentNode2 != null ? parentNode2.jsonContextNode : null,
- link != null ? link.property.id : null,
- cell,
- rowIndex
- );
- }
- return tnode;
- }
-
- @Override
- public TransposedNode transposeTopicNode(
- TransposedNode parentNode,
- Link link,
- FreebaseTopicNode node,
- int rowIndex) {
-
- WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
- WritingTransposedNode tnode = new TopicTransposedNode(node);
-
- tnode.load =
- (parentNode2 == null || parentNode2.load) &&
- (link == null || link.load);
-
- processTransposedNode(tnode, parentNode, link, rowIndex);
-
- tnode.jsonContextNode = addJsonContext(
- parentNode2 != null ? parentNode2.jsonContextNode : null,
- link != null ? link.property.id : null,
- node.topic
- );
-
- return tnode;
- }
-
- @Override
- public TransposedNode transposeValueNode(
- TransposedNode parentNode,
- Link link,
- ValueNode node,
- int rowIndex) {
-
- WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
- WritingTransposedNode tnode = new ValueTransposedNode(node);
-
- tnode.load =
- (parentNode2 == null || parentNode2.load) &&
- (link == null || link.load);
-
- processTransposedNode(tnode, parentNode, link, rowIndex);
-
- tnode.jsonContextNode = addJsonContext(
- parentNode2 != null ? parentNode2.jsonContextNode : null,
- link != null ? link.property.id : null,
- node.value
- );
-
- return tnode;
- }
-
- protected void processTransposedNode(
- WritingTransposedNode tnode,
- TransposedNode parentNode,
- Link link,
- int rowIndex
- ) {
- if (parentNode != null) {
- if (parentNode instanceof TransposedNodeWithChildren) {
- TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
- parentNode2.rowIndices.add(rowIndex);
- parentNode2.children.add(tnode);
- parentNode2.links.add(link);
- }
- } else {
- addRootNode(tnode, rowIndex);
- }
- }
-
- protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
- JSONObject o = new JSONObject();
-
- try {
- if (value instanceof FreebaseTopic) {
- FreebaseTopic topic = (FreebaseTopic) value;
- o.put("id", topic.id);
- o.put("name", topic.name);
- } else {
- o.put("v", value);
- }
- } catch (JSONException e) {
- // ignore
- }
-
- connectJsonContext(parent, o, key);
- return o;
- }
-
- protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
- JSONObject o = new JSONObject();
-
- connectJsonContext(parent, o, key);
-
- try {
- if (cell != null) {
- o.put("v", cell.value);
- if (cell.recon != null) {
- o.put("recon", "rec" + cell.recon.id);
-
- if (cell.recon.judgment == Judgment.Matched) {
- o.put("id", cell.recon.match.id);
- o.put("name", cell.recon.match.name);
- }
-
- // qa:display_context
- {
- StringBuffer sb2 = new StringBuffer();
-
- sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
- sb2.append(Long.toString(cell.recon.id));
- sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
- sb2.append(Long.toString(contextID));
- sb2.append("\", \"meta\" : { \"row\" : ");
- sb2.append(Integer.toString(rowIndex));
- sb2.append(" } }");
-
- writeLine(sb2.toString());
- }
- }
- }
- } catch (JSONException e) {
- // ignore
- }
-
- return o;
- }
-
- protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
- try {
- if (parent == null) {
- contextTreeRoot = o;
- } else {
- JSONArray a = null;
- if (parent.has(key)) {
- a = parent.getJSONArray(key);
- } else {
- a = new JSONArray();
- parent.put(key, a);
- }
-
- a.put(o);
- }
- } catch (JSONException e) {
- // ignore
- }
- }
-
- protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
- if (lastRootNode != null) {
- lastRootNode.write(null, null, project, -1, -1, null);
- writeContextTreeNode();
- }
- lastRootNode = tnode;
-
- contextTreeRoot = null;
- contextRowIndex = rowIndex;
- contextRefCount = 0;
- contextID++;
- }
-
- protected void writeContextTreeNode() {
- if (contextTreeRoot != null && contextRefCount > 0) {
- StringBuffer sb = new StringBuffer();
-
- sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
- sb.append(Long.toString(contextID));
- sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
- sb.append(Integer.toString(contextRowIndex));
- sb.append(", \"data\" : ");
- sb.append(contextTreeRoot.toString());
- sb.append(" } }");
-
- writeLine(sb.toString());
- }
- }
-
- static protected Object validateValue(Object value, String valueType) {
- if ("/type/datetime".equals(valueType)) {
- if (value instanceof Calendar || value instanceof Date) {
- DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
- value = formatter.format(value instanceof Date ? ((Date) value)
- : ((Calendar) value).getTime());
- } else if (!(value instanceof String)) {
- value = value.toString();
- }
- } else if ("/type/boolean".equals(valueType)) {
- if (!(value instanceof Boolean)) {
- value = Boolean.parseBoolean(value.toString());
- }
- } else if ("/type/int".equals(valueType)) {
- if (value instanceof Number) {
- value = ((Number) value).longValue();
- } else {
- try {
- value = Long.parseLong(value.toString());
- } catch (NumberFormatException e) {
- value = null;
- }
- }
- } else if ("/type/float".equals(valueType)) {
- if (value instanceof Number) {
- value = ((Number) value).floatValue();
- } else {
- try {
- value = Float.parseFloat(value.toString());
- } catch (NumberFormatException e) {
- value = null;
- }
- }
- } else if ("/type/double".equals(valueType)) {
- if (value instanceof Number) {
- value = ((Number) value).doubleValue();
- } else {
- try {
- value = Double.parseDouble(value.toString());
- } catch (NumberFormatException e) {
- value = null;
- }
- }
- }
-
- return value;
- }
-}
diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
deleted file mode 100644
index 5fa31a35f..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
-
-Copyright 2010,2013 Google Inc. and contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.util;
-
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.google.refine.freebase.FreebaseType;
-import com.google.refine.model.ReconCandidate;
-import com.google.refine.util.JSONUtilities;
-import com.google.refine.util.ParsingUtilities;
-
-public class FreebaseDataExtensionJob {
- static public class DataExtension {
- final public Object[][] data;
-
- public DataExtension(Object[][] data) {
- this.data = data;
- }
- }
-
- static public class ColumnInfo {
- final public List names;
- final public List path;
- final public FreebaseType expectedType;
-
- protected ColumnInfo(List names, List path, FreebaseType expectedType) {
- this.names = names;
- this.path = path;
- this.expectedType = expectedType;
- }
- }
-
- final public JSONObject extension;
- final public int columnCount;
- final public List columns = new ArrayList();
-
- public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
- this.extension = obj;
- this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ?
- countColumns(obj.getJSONArray("properties"), columns, new ArrayList(), new ArrayList()) : 0;
- }
-
- public Map extend(
- Set ids,
- Map reconCandidateMap
- ) throws Exception {
- StringWriter writer = new StringWriter();
- formulateQuery(ids, extension, writer);
- String query = writer.toString();
-
- String result = FreebaseUtils.mqlread(query);
-
- JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result);
- Map map = new HashMap();
- if (o.has("result")) {
- JSONArray a = o.getJSONArray("result");
- int l = a.length();
-
- for (int i = 0; i < l; i++) {
- JSONObject o2 = a.getJSONObject(i);
- String id = o2.getString("id");
- FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
-
- if (ext != null) {
- map.put(id, ext);
- }
- }
- }
-
- return map;
- }
-
-
- protected FreebaseDataExtensionJob.DataExtension collectResult(
- JSONObject obj,
- Map reconCandidateMap
- ) throws JSONException {
- List rows = new ArrayList();
-
- collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
-
- Object[][] data = new Object[rows.size()][columnCount];
- rows.toArray(data);
-
- return new DataExtension(data);
- }
-
- protected void storeKey(
- List rows,
- int row,
- int col,
- JSONObject key,
- Map reconCandidateMap
- ) throws JSONException {
- String keyval = key.getString("value");
- while (row >= rows.size()) {
- rows.add(new Object[columnCount]);
- }
- rows.get(row)[col] = keyval;
- }
-
- protected void storeCell(
- List rows,
- int row,
- int col,
- Object value,
- Map reconCandidateMap
- ) {
- while (row >= rows.size()) {
- rows.add(new Object[columnCount]);
- }
- rows.get(row)[col] = value;
- }
-
- protected void storeCell(
- List rows,
- int row,
- int col,
- JSONObject obj,
- Map reconCandidateMap
- ) throws JSONException {
- String id = obj.getString("id");
- ReconCandidate rc;
- if (reconCandidateMap.containsKey(id)) {
- rc = reconCandidateMap.get(id);
- } else {
- rc = new ReconCandidate(
- obj.getString("id"),
- obj.getString("name"),
- JSONUtilities.getStringArray(obj, "type"),
- 100
- );
-
- reconCandidateMap.put(id, rc);
- }
-
- storeCell(rows, row, col, rc, reconCandidateMap);
- }
-
- protected int[] collectResult(
- List rows,
- JSONObject extNode,
- JSONObject resultNode,
- int startRowIndex,
- int startColumnIndex,
- Map reconCandidateMap
- ) throws JSONException {
- String propertyID = extNode.getString("id");
- String expectedTypeID = extNode.getJSONObject("expected").getString("id");
-
- JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ?
- resultNode.getJSONArray(propertyID) : null;
-
- if ("/type/key".equals(expectedTypeID)) {
- if (a != null) {
- int l = a.length();
- for (int r = 0; r < l; r++) {
- Object o = a.isNull(r) ? null : a.get(r);
- if (o instanceof JSONObject) {
- storeKey(rows, startRowIndex++, startColumnIndex, (JSONObject) o, reconCandidateMap);
- }
- }
- }
-
- // note that we still take up a column even if we don't have any data
- return new int[] { startRowIndex, startColumnIndex + 1 };
- } else if (expectedTypeID.startsWith("/type/")) {
- if (a != null) {
- int l = a.length();
- for (int r = 0; r < l; r++) {
- Object o = a.isNull(r) ? null : a.get(r);
- if (o instanceof Serializable) {
- storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap);
- }
- }
- }
-
- // note that we still take up a column even if we don't have any data
- return new int[] { startRowIndex, startColumnIndex + 1 };
- } else {
- boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
- boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
-
- if (a != null && a.length() > 0) {
- int maxColIndex = startColumnIndex;
-
- int l = a.length();
- for (int r = 0; r < l; r++) {
- Object v = a.isNull(r) ? null : a.get(r);
- JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null;
-
- int startColumnIndex2 = startColumnIndex;
- int startRowIndex2 = startRowIndex;
-
- if (isOwnColumn) {
- if (o != null) {
- storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap);
- } else {
- storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap);
- }
- }
-
- if (hasSubProperties && o != null) {
- int[] rowcol = collectResult(
- rows,
- extNode.getJSONArray("properties"),
- o,
- startRowIndex,
- startColumnIndex2,
- reconCandidateMap
- );
-
- startRowIndex2 = rowcol[0];
- startColumnIndex2 = rowcol[1];
- }
-
- startRowIndex = startRowIndex2;
- maxColIndex = Math.max(maxColIndex, startColumnIndex2);
- }
-
- return new int[] { startRowIndex, maxColIndex };
- } else {
- return new int[] {
- startRowIndex,
- startColumnIndex + countColumns(extNode, null, new ArrayList(), new ArrayList())
- };
- }
- }
- }
-
- protected int[] collectResult(
- List rows,
- JSONArray subProperties,
- JSONObject resultNode,
- int startRowIndex,
- int startColumnIndex,
- Map reconCandidateMap
- ) throws JSONException {
- int maxStartRowIndex = startRowIndex;
-
- int k = subProperties.length();
- for (int c = 0; c < k; c++) {
- int[] rowcol = collectResult(
- rows,
- subProperties.getJSONObject(c),
- resultNode,
- startRowIndex,
- startColumnIndex,
- reconCandidateMap
- );
-
- maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]);
- startColumnIndex = rowcol[1];
- }
-
- return new int[] { maxStartRowIndex, startColumnIndex };
- }
-
-
-
- static protected void formulateQuery(Set ids, JSONObject node, Writer writer) throws JSONException {
- JSONWriter jsonWriter = new JSONWriter(writer);
-
- jsonWriter.array();
- jsonWriter.object();
-
- jsonWriter.key("id"); jsonWriter.value(null);
- jsonWriter.key("id|=");
- jsonWriter.array();
- for (String id : ids) {
- if (id != null) {
- jsonWriter.value(id);
- }
- }
- jsonWriter.endArray();
-
- formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
-
- jsonWriter.endObject();
- jsonWriter.endArray();
- }
-
- static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
- String propertyID = node.getString("id");
- String expectedTypeID = node.getJSONObject("expected").getString("id");
-
- writer.key(propertyID);
- writer.array();
- {
- if (!expectedTypeID.startsWith("/type/") // not literal
- || "/type/key".equals(expectedTypeID)) {
- writer.object();
- writer.key("optional"); writer.value(true);
-
- boolean hasLimit = false;
- if (node.has("constraints") && !node.isNull("constraints")) {
- JSONObject constraints = node.getJSONObject("constraints");
-
- String[] names = JSONObject.getNames(constraints);
- for (String name : names) {
- Object value = constraints.get(name);
- if (name.equals("limit")) {
- hasLimit = true;
- }
-
- if (!name.contains(":") &&
- !name.equals("limit") &&
- !name.equals("optional") &&
- !name.equals("count") &&
- !name.equals("estimate-count") &&
- !name.equals("sort") &&
- !name.equals("return")) {
-
- if (name.startsWith("!")) {
- name = "!c:" + name.substring(1);
- } else {
- name = "c:" + name;
- }
- }
- if (name.equals("sort")) {
- String sortKey = (String) value;
- if (sortKey.startsWith("-")) {
- sortKey = sortKey.substring(1);
- }
- writer.key(sortKey); writer.value(null);
- writer.key(name); writer.value(value);
- } else {
- writer.key(name); writer.value(value);
- }
- }
- }
- if (!hasLimit) {
- writer.key("limit"); writer.value(10);
- }
-
- {
- boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
-
- if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
- if ("/type/key".equals(expectedTypeID)) {
- writer.key("value"); writer.value(null);
- } else {
- writer.key("name"); writer.value(null);
- writer.key("id"); writer.value(null);
- }
- writer.key("type"); writer.array(); writer.endArray();
- }
-
- if (hasSubProperties) {
- formulateQueryNode(node.getJSONArray("properties"), writer);
- }
- }
- writer.endObject();
- }
- }
- writer.endArray();
- }
-
- static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
- int l = propertiesA.length();
-
- for (int i = 0; i < l; i++) {
- formulateQueryNode(propertiesA.getJSONObject(i), writer);
- }
- }
-
- static protected int countColumns(JSONObject obj, List columns, List names, List path) throws JSONException {
- String name = obj.getString("name");
-
- List names2 = null;
- List path2 = null;
- if (columns != null) {
- names2 = new ArrayList(names);
- names2.add(name);
-
- path2 = new ArrayList(path);
- path2.add(obj.getString("id"));
- }
-
- if (obj.has("properties") && !obj.isNull("properties")) {
- boolean included = (obj.has("included") && obj.getBoolean("included"));
- if (included && columns != null) {
- JSONObject expected = obj.getJSONObject("expected");
-
- columns.add(new ColumnInfo(names2, path2,
- new FreebaseType(expected.getString("id"), expected.getString("name"))));
- }
-
- return (included ? 1 : 0) +
- countColumns(obj.getJSONArray("properties"), columns, names2, path2);
- } else {
- if (columns != null) {
- JSONObject expected = obj.getJSONObject("expected");
-
- columns.add(new ColumnInfo(names2, path2,
- new FreebaseType(expected.getString("id"), expected.getString("name"))));
- }
- return 1;
- }
- }
-
- static protected int countColumns(JSONArray a, List columns, List names, List path) throws JSONException {
- int c = 0;
- int l = a.length();
- for (int i = 0; i < l; i++) {
- c += countColumns(a.getJSONObject(i), columns, names, path);
- }
- return c;
- }
-}
\ No newline at end of file
diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
deleted file mode 100644
index e7124a056..000000000
--- a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
+++ /dev/null
@@ -1,543 +0,0 @@
-/*
-
-Copyright 2010,2013 Google Inc. and other contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-*/
-
-package com.google.refine.freebase.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.UnsupportedEncodingException;
-import java.io.Writer;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.servlet.http.HttpServletRequest;
-
-import oauth.signpost.OAuthConsumer;
-import oauth.signpost.exception.OAuthCommunicationException;
-import oauth.signpost.exception.OAuthExpectationFailedException;
-import oauth.signpost.exception.OAuthMessageSignerException;
-
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.params.CoreProtocolPNames;
-import org.apache.http.util.EntityUtils;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.api.client.googleapis.batch.BatchRequest;
-import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
-import com.google.api.client.googleapis.json.GoogleJsonError;
-import com.google.api.client.http.HttpHeaders;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.freebase.Freebase;
-import com.google.api.services.freebase.FreebaseRequestInitializer;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.RefineServlet;
-import com.google.refine.oauth.Credentials;
-import com.google.refine.oauth.OAuthUtilities;
-import com.google.refine.oauth.Provider;
-import com.google.refine.preference.PreferenceStore;
-import com.google.refine.util.ParsingUtilities;
-
-public class FreebaseUtils {
-
- private static final String FREEBASE_API_VERSION = "v1";
-// private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox";
-
- private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc";
-
- private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION;
-
- private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch";
-
- static final Logger logger = LoggerFactory.getLogger("freebase");
-
- static final public String FREEBASE_HOST = "freebase.com";
-
- static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
-
- static final private String AGENT_ID = "/en/google_refine";
-
- static final private int SAMPLE_SIZE = 300;
- static final private int JUDGES = 4;
-
- public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
-
- public static String getApiKey() {
- PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
- String key = (String) ps.get("freebase.api.key");
- if (key == null) {
- key = System.getProperty("refine.google_api_key");
- }
- return key == null ? API_KEY : key;
- }
-
- private static String getUserInfoURL(String host) {
- // TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
- return "http://api." + host + "/api/service/user_info";
- }
-
- private static String getMQLWriteURL(String host) {
- return "http://api." + host + "/api/service/mqlwrite";
- }
-
- private static String getMQLReadURL(String host) {
- return "http://api." + host + "/api/service/mqlread";
- }
-
- private static String getUserAgent() {
- return RefineServlet.FULLNAME;
- }
-
- public static String getUserInfo(Credentials credentials, Provider provider)
- throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
-
- OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
-
- HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
-
- // this is required by the Metaweb API to avoid XSS
- httpRequest.setHeader("X-Requested-With", "1");
-
- // sign the request with the oauth library
- consumer.sign(httpRequest);
-
- // execute the request
- HttpClient httpClient = new DefaultHttpClient();
- HttpResponse httpResponse = httpClient.execute(httpRequest);
-
- // return the results
- return EntityUtils.toString(httpResponse.getEntity());
- }
-
- public static String getUserBadges(Provider provider, String user_id)
- throws ClientProtocolException, IOException, JSONException {
-
- String query = "{" +
- "'id' : '" + user_id + "'," +
- "'!/type/usergroup/member' : [{" +
- "'id' : null," +
- "'key' : [{" +
- "'namespace' : null" +
- "}]" +
- "}]" +
- "}".replace("'", "\"");
-
- return mqlread(provider, query);
- }
-
- /**
- * Perform an MQLREAD operation using the credentials of the given OAuth provider
- *
- * @deprecated This will go away when we switch to Google authentication.
- */
- @Deprecated
- public static String mqlread(Provider provider, String query)
- throws ClientProtocolException, IOException, JSONException {
-
- JSONObject envelope = new JSONObject();
- envelope.put("query", new JSONObject(query));
-
- List formparams = new ArrayList();
- formparams.add(new BasicNameValuePair("query", envelope.toString()));
- UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
-
- HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
- httpRequest.setEntity(entity);
-
- // this is required by the Metaweb API to avoid XSS
- httpRequest.setHeader("X-Requested-With", "1");
-
- // execute the request
- HttpClient httpClient = new DefaultHttpClient();
- HttpResponse httpResponse = httpClient.execute(httpRequest);
-
- // return the results
- return EntityUtils.toString(httpResponse.getEntity());
- }
-
-
- /**
- * Perform a single unauthenticated MQLread.
- *
- * (wrapper method for a bunch of alternative implementations)
- */
- static public String mqlread(String query)
- throws IOException, JSONException {
- // A bunch of implementations which don't work for MQLread, but do for other methods
- // String result = rpcCall(query);
- // String result = googleCall(query);
- // String result = batchCall1(query);
-
- String result = mqlreadBatchMime(query);
- return result;
- }
-
-
- public static String mqlwrite(Credentials credentials, Provider provider, String query)
- throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
- OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
-
- JSONObject envelope = new JSONObject();
- envelope.put("query", new JSONObject(query));
-
- List formparams = new ArrayList();
- formparams.add(new BasicNameValuePair("query", envelope.toString()));
- UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
-
- HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
- httpRequest.setEntity(entity);
-
- // this is required by the Metaweb API to avoid XSS
- httpRequest.setHeader("X-Requested-With", "1");
-
- // sign the request with the oauth library
- consumer.sign(httpRequest);
-
- // execute the request
- HttpClient httpClient = new DefaultHttpClient();
- HttpResponse httpResponse = httpClient.execute(httpRequest);
-
- // return the results
- return EntityUtils.toString(httpResponse.getEntity());
- }
-
- private static String getTweezersParams(int sample_size, int judges) {
- String o = "{" +
- "'sample_size':" + sample_size + "," +
- "'votes':{" +
- "'reconciled':" + judges + "," +
- "'invalid':" + judges + "," +
- "'new':" + judges + "," +
- "'skip':" + (judges + 2) +
- "}" +
- "}";
- return o.replace('\'', '"');
- }
-
- public static String uploadTriples(
- HttpServletRequest request,
- String qa,
- String source_name,
- String source_id,
- String mdo_id,
- String triples
- ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
-
- Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
-
- Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
-
- JSONObject mdo_info = new JSONObject();
- mdo_info.put("name", source_name);
- if (source_id != null) {
- mdo_info.put("info_source",source_id);
- }
-
- JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
- if (user_info.has("username")) {
-
- List formparams = new ArrayList();
- formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
- formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
- formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
- formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
- formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
- formparams.add(new BasicNameValuePair("graphport", "sandbox"));
- formparams.add(new BasicNameValuePair("payload", triples));
- formparams.add(new BasicNameValuePair("check_params", "false"));
- if (mdo_id != null) {
- formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
- }
- if (Boolean.parseBoolean(qa)) {
- formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
- }
-
- String freeqKey = System.getProperty("freeq.key");
- if (freeqKey != null) {
- logger.warn("Found Freeq key, will bypass OAuth signature");
- formparams.add(new BasicNameValuePair("apikey", freeqKey));
- }
-
- UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
-
- HttpPost httpRequest = new HttpPost(getFreeQUrl());
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
- httpRequest.setEntity(entity);
-
- if (freeqKey == null) {
- logger.warn("Calculating OAuth signature");
- HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
- surrogateRequest.setEntity(entity);
-
- OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
-
- // TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
- // Is there something we can do to generate an oauth signature without consuming so much memory?
- consumer.sign(surrogateRequest);
-
- Header[] h = surrogateRequest.getHeaders("Authorization");
- if (h.length > 0) {
- httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
- } else {
- throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
- }
- }
-
- // execute the request
- HttpClient httpClient = new DefaultHttpClient();
- HttpResponse httpResponse = httpClient.execute(httpRequest);
-
- // return the results
- return EntityUtils.toString(httpResponse.getEntity());
- } else {
- throw new RuntimeException("Invalid credentials");
- }
- }
-
- static public String getFreeQUrl() {
- String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
- return url != null ? url : FREEQ_URL;
- }
-
-
- static final String BOUNDARY = "---theOpenRefineBoundary--=";
-
- /**
- * A hand rolled MIME multipart/mixed implementation for Google's Batch API
- */
- static private String mqlreadBatchMime(String query) throws JSONException, IOException {
- URL url = new URL(GOOGLE_BATCH_URL);
- String service_url = FREEBASE_SERVICE_URL+"/mqlread";
-
- // We could use the javax.mail package, but it's actually more trouble than it's worth
- String body = "--" + BOUNDARY + "\n"
- + queryToMimeBodyPart("0", query, service_url, FreebaseUtils.getApiKey())
- + "\n--" + BOUNDARY + "\n" ;
-
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY);
- connection.setConnectTimeout(5000);
- connection.setDoOutput(true);
-
- Writer writer = new OutputStreamWriter(connection.getOutputStream());
- try {
- writer.write(body);
- } finally {
- writer.flush();
- writer.close();
- }
-
- connection.connect();
- String result = null;
- if (connection.getResponseCode() >= 400) {
- String responseMessage = connection.getResponseMessage();
- String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
- LoggerFactory.getLogger("freebase").error(
- "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
- + errorStream);
- } else {
- InputStream is = connection.getInputStream();
- try {
- String s = ParsingUtilities.inputStreamToString(is);
- String boundary = s.substring(0,s.indexOf("\n"));
- boundary = boundary.split("\r")[0];
- String[] part = s.split(boundary); // part 0 is empty because of leading boundary
- String[] sections = part[1].split("\r\n\r\n");
- // Mime headers, followed by HTTP headers, followd by actual response
- result = sections[2];
- } finally {
- is.close();
- }
- }
- return result;
- }
-
- static String queryToMimeBodyPart(String query_name,
- String query, String service_url, String api_key)
- throws IOException {
- // We could use the javax.mail package, but it's actually more trouble than it's worth
- StringBuilder sb = new StringBuilder();
- sb.append("Content-Type: application/http\n");
- sb.append("Content-Transfer-Encoding: binary\n");
- sb.append("Content-ID: " + query_name + "\n");
- sb.append("\n");
-
- List params = new ArrayList();
- params.add(new BasicNameValuePair("query",query));
- params.add(new BasicNameValuePair("key", api_key));
- UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
-
- String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
- sb.append(body);
- sb.append("\n");
-
- return sb.toString();
- }
-
- //////////////////////// Unused methods for future use /////////////////////
-
- /**
- * This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
- *
- * NOTE: JSONRPC has been deprecated and replaced by HTTP Batch (which also
- * doesn't support MQLread, so perhaps we should just remove this))
- */
- @SuppressWarnings("unused")
- static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {
- URL url = new URL(GOOGLE_RPC_URL);
-
- JSONObject params = new JSONObject();
- params.put("query",query);
- params.put("key", FreebaseUtils.getApiKey());
-
- JSONObject req1 = new JSONObject();
- req1.put("jsonrpc","2.0");
- req1.put("id","q0");
- req1.put("method","freebase.mqlread");
- req1.put("apiVersion", FREEBASE_API_VERSION);
- req1.put("params",params);
-
- JSONArray body = new JSONArray();
- body.put(req1);
-
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setRequestProperty("Content-Type", "application/json"); //
- connection.setConnectTimeout(5000);
- connection.setDoOutput(true);
-
- OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
- try {
- writer.write(body.toString());
- } finally {
- writer.flush();
- writer.close();
- }
-
- connection.connect();
- JSONArray result = null;
- if (connection.getResponseCode() >= 400) {
- String responseMessage = connection.getResponseMessage();
- String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
- LoggerFactory.getLogger("freebase").error(
- "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
- + errorStream);
- } else {
- InputStream is = connection.getInputStream();
- try {
- String s = ParsingUtilities.inputStreamToString(is);
- result = ParsingUtilities.evaluateJsonStringToArray(s);
- } finally {
- is.close();
- }
- }
- return result.getJSONObject(0);
- }
-
-
- private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
- private static final FreebaseRequestInitializer REQUEST_INITIALIZER =
- new FreebaseRequestInitializer(FreebaseUtils.getApiKey());
-
- /**
- * Submit a single MQL read query via the standard Google client library
- */
- @SuppressWarnings("unused")
- static private String mqlreadFreebaseClient(String query)
- throws IOException, JSONException {
-
- Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
- .setApplicationName("OpenRefine")
- .setFreebaseRequestInitializer(REQUEST_INITIALIZER)
- .build();
-
- InputStream is = client.mqlread(query).executeAsInputStream();
- String result = ParsingUtilities.inputStreamToString(is);
- return result;
- }
-
-
- /**
- * Submit a single MQL query via the Batch endpoint
- * (not supported by Google's Java client)
- */
- @SuppressWarnings("unused")
- static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException {
- JSONObject response = null;
-
- // FIXME: We really want JsonBatchCallback here, but it's not supported right now
- JsonBatchCallback callback = new JsonBatchCallback() {
- public void onSuccess(Void res, HttpHeaders responseHeaders) {
- System.out.println(res);
- }
- public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
- System.out.println("Error Message: " + e.getMessage());
- }
- };
-
- Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
- .setApplicationName("OpenRefine")
- .setFreebaseRequestInitializer(REQUEST_INITIALIZER)
- .build();
-
- // FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest
- BatchRequest batch = client.batch();
- client.mqlread(query).queue(batch, callback);
- batch.execute();
-
- return response;
- }
-
-}
diff --git a/extensions/jython/src/com/google/refine/jython/JythonEvaluable.java b/extensions/jython/src/com/google/refine/jython/JythonEvaluable.java
index 2b96794c8..70bcfdb53 100644
--- a/extensions/jython/src/com/google/refine/jython/JythonEvaluable.java
+++ b/extensions/jython/src/com/google/refine/jython/JythonEvaluable.java
@@ -33,29 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.jython;
+import com.google.refine.expr.*;
+import org.python.core.*;
+import org.python.util.PythonInterpreter;
+
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
-import org.python.core.Py;
-import org.python.core.PyException;
-import org.python.core.PyFloat;
-import org.python.core.PyFunction;
-import org.python.core.PyInteger;
-import org.python.core.PyLong;
-import org.python.core.PyNone;
-import org.python.core.PyObject;
-import org.python.core.PyString;
-import org.python.util.PythonInterpreter;
-
-import com.google.refine.expr.EvalError;
-import com.google.refine.expr.Evaluable;
-import com.google.refine.expr.HasFields;
-import com.google.refine.expr.LanguageSpecificParser;
-import com.google.refine.expr.ParsingException;
-
public class JythonEvaluable implements Evaluable {
static public LanguageSpecificParser createParser() {
@@ -68,7 +55,7 @@ public class JythonEvaluable implements Evaluable {
};
}
- private static final String s_functionName = "___temp___";
+ private final String s_functionName;
private static PythonInterpreter _engine;
@@ -97,6 +84,8 @@ public class JythonEvaluable implements Evaluable {
}
public JythonEvaluable(String s) {
+ this.s_functionName = String.format("__temp_%d__", Math.abs(s.hashCode()));
+
// indent and create a function out of the code
String[] lines = s.split("\r\n|\r|\n");
diff --git a/extensions/jython/tests/com/google/refine/jython/JythonEvaluableTest.java b/extensions/jython/tests/com/google/refine/jython/JythonEvaluableTest.java
new file mode 100644
index 000000000..a9aec0993
--- /dev/null
+++ b/extensions/jython/tests/com/google/refine/jython/JythonEvaluableTest.java
@@ -0,0 +1,47 @@
+package com.google.refine.jython;
+
+import com.google.refine.expr.CellTuple;
+import com.google.refine.expr.Evaluable;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Row;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.Properties;
+
+/**
+ * @author Maxim Galushka
+ */
+public class JythonEvaluableTest {
+
+ @Test
+ public void testJythonConcurrent(){
+ Properties props = new Properties();
+ Project project = new Project();
+
+ Row row = new Row(2);
+ row.setCell(0, new Cell("one", null));
+ row.setCell(0, new Cell("1", null));
+
+ props.put("columnName", "number");
+ props.put("true", "true");
+ props.put("false", "false");
+ props.put("rowIndex", "0");
+ props.put("value", 1);
+ props.put("project", project);
+ props.put("call", "number");
+ props.put("PI", "3.141592654");
+ props.put("cells", new CellTuple(project, row));
+
+ Evaluable eval1 = new JythonEvaluable("a = value\nreturn a * 2");
+ Long value1 = (Long) eval1.evaluate(props);
+
+ // create some unrelated evaluable
+ new JythonEvaluable("a = value\nreturn a * 10");
+
+ // repeat same previous test
+ Long value2 = (Long) eval1.evaluate(props);
+ Assert.assertEquals(value1, value2);
+ }
+}
diff --git a/main/src/com/google/refine/browsing/Engine.java b/main/src/com/google/refine/browsing/Engine.java
index f8cffce53..64b2a0432 100644
--- a/main/src/com/google/refine/browsing/Engine.java
+++ b/main/src/com/google/refine/browsing/Engine.java
@@ -91,7 +91,7 @@ public class Engine implements Jsonizable {
_mode = mode;
}
- public FilteredRows getAllRows() {
+ public synchronized FilteredRows getAllRows() {
return new FilteredRows() {
@Override
public void accept(Project project, RowVisitor visitor) {
@@ -134,7 +134,7 @@ public class Engine implements Jsonizable {
throw new InternalError("Unknown mode.");
}
- public FilteredRecords getAllRecords() {
+ public synchronized FilteredRecords getAllRecords() {
return new FilteredRecords() {
@Override
public void accept(Project project, RecordVisitor visitor) {
diff --git a/main/src/com/google/refine/exporters/HtmlTableExporter.java b/main/src/com/google/refine/exporters/HtmlTableExporter.java
index 2f14ba12c..1c4163ee2 100644
--- a/main/src/com/google/refine/exporters/HtmlTableExporter.java
+++ b/main/src/com/google/refine/exporters/HtmlTableExporter.java
@@ -61,9 +61,12 @@ public class HtmlTableExporter implements WriterExporter {
public void startFile(JSONObject options) {
try {
writer.write("\n");
- writer.write("");
+ writer.write("<head>\n");
+ writer.write("<title>");
writer.write(ProjectManager.singleton.getProjectMetadata(project.id).getName());
- writer.write(" \n");
+ writer.write("\n");
+ writer.write("\n");
diff --git a/main/src/com/google/refine/expr/MetaParser.java b/main/src/com/google/refine/expr/MetaParser.java
index 12b5cd8a3..a9d8d8ae5 100644
--- a/main/src/com/google/refine/expr/MetaParser.java
+++ b/main/src/com/google/refine/expr/MetaParser.java
@@ -120,6 +120,14 @@ abstract public class MetaParser {
}, "value");
}
+ /**
+ * languagePrefix will be stored in the meta model as an identifier.
+ * so be careful when change it as it will break the backward compatibility for the old project
+ * @param languagePrefix
+ * @param name
+ * @param parser
+ * @param defaultExpression
+ */
static public void registerLanguageParser(String languagePrefix, String name, LanguageSpecificParser parser, String defaultExpression) {
s_languages.put(languagePrefix, new LanguageInfo(name, parser, defaultExpression));
}
diff --git a/main/src/com/google/refine/importers/ExcelImporter.java b/main/src/com/google/refine/importers/ExcelImporter.java
index 802344bcc..4de1b3369 100644
--- a/main/src/com/google/refine/importers/ExcelImporter.java
+++ b/main/src/com/google/refine/importers/ExcelImporter.java
@@ -37,12 +37,14 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PushbackInputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLException;
import org.apache.poi.common.usermodel.Hyperlink;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
@@ -77,10 +79,7 @@ public class ExcelImporter extends TabularImportingParserBase {
public JSONObject createParserUIInitializationData(
ImportingJob job, List fileRecords, String format) {
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
-
- boolean xmlBased = "text/xml/xlsx".equals(format);
- JSONUtilities.safePut(options, "xmlBased", xmlBased);
-
+
JSONArray sheetRecords = new JSONArray();
JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
try {
@@ -88,8 +87,13 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file);
+
+ if (!is.markSupported()) {
+ is = new PushbackInputStream(is, 8);
+ }
+
try {
- Workbook wb = xmlBased ?
+ Workbook wb = POIXMLDocument.hasOOXMLHeader(is) ?
new XSSFWorkbook(is) :
new HSSFWorkbook(new POIFSFileSystem(is));
@@ -136,10 +140,13 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONObject options,
List exceptions
) {
- boolean xmlBased = JSONUtilities.getBoolean(options, "xmlBased", false);
Workbook wb = null;
+ if (!inputStream.markSupported()) {
+ inputStream = new PushbackInputStream(inputStream, 8);
+ }
+
try {
- wb = xmlBased ?
+ wb = POIXMLDocument.hasOOXMLHeader(inputStream) ?
new XSSFWorkbook(inputStream) :
new HSSFWorkbook(new POIFSFileSystem(inputStream));
} catch (IOException e) {
diff --git a/main/src/com/google/refine/importers/MarcImporter.java b/main/src/com/google/refine/importers/MarcImporter.java
index 9ffa6ea4b..5d561483e 100644
--- a/main/src/com/google/refine/importers/MarcImporter.java
+++ b/main/src/com/google/refine/importers/MarcImporter.java
@@ -41,6 +41,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import org.json.JSONObject;
+import org.marc4j.MarcException;
import org.marc4j.MarcPermissiveStreamReader;
import org.marc4j.MarcWriter;
import org.marc4j.MarcXmlWriter;
@@ -58,14 +59,11 @@ public class MarcImporter extends XmlImporter {
@Override
public JSONObject createParserUIInitializationData(ImportingJob job, java.util.List fileRecords, String format) {
-
if (fileRecords.size() > 0) {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
File tempFile = new File(file.getAbsolutePath()+".xml");
- JSONUtilities.safePut(firstFileRecord, "location",
- JSONUtilities.getString(firstFileRecord, "location", "")+".xml");
try {
InputStream inputStream = new FileInputStream(file);
@@ -84,7 +82,14 @@ public class MarcImporter extends XmlImporter {
try {
outputStream.close();
inputStream.close();
- file.delete(); // get rid of our original file
+
+ if (tempFile.length() == 0) // write failed. Most of time because of wrong Marc format
+ tempFile.delete();
+ else // only set json if write the temp file successfully:
+ JSONUtilities.safePut(firstFileRecord, "location",
+ JSONUtilities.getString(firstFileRecord, "location", "")+".xml");
+
+// file.delete(); // get rid of our original file
} catch (IOException e) {
// Just ignore - not much we can do anyway
}
@@ -93,11 +98,8 @@ public class MarcImporter extends XmlImporter {
logger.error("Failed to create temporary XML file from MARC file", e);
}
}
-
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
-
return options;
};
-
}
diff --git a/main/src/com/google/refine/importers/tree/ImportParameters.java b/main/src/com/google/refine/importers/tree/ImportParameters.java
new file mode 100644
index 000000000..141bcdcfb
--- /dev/null
+++ b/main/src/com/google/refine/importers/tree/ImportParameters.java
@@ -0,0 +1,28 @@
+package com.google.refine.importers.tree;
+
+
+public class ImportParameters {
+ boolean trimStrings;
+ boolean storeEmptyStrings;
+ boolean guessDataType;
+ boolean includeFileSources;
+ String fileSource;
+
+ public ImportParameters(boolean trimStrings, boolean storeEmptyStrings, boolean guessCellValueTypes,
+ boolean includeFileSources, String fileSource) {
+ this.trimStrings = trimStrings;
+ this.storeEmptyStrings = storeEmptyStrings;
+ this.guessDataType = guessCellValueTypes;
+ this.includeFileSources = includeFileSources;
+ this.fileSource = fileSource;
+ }
+
+ public ImportParameters(boolean trimStrings, boolean storeEmptyStrings, boolean guessCellValueTypes) {
+ this.trimStrings = trimStrings;
+ this.storeEmptyStrings = storeEmptyStrings;
+ this.guessDataType = guessCellValueTypes;
+ this.includeFileSources = false;
+ this.fileSource = "";
+ }
+
+}
\ No newline at end of file
diff --git a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
index 866657d94..dcf7b3d5d 100644
--- a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
+++ b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
@@ -48,7 +48,11 @@ import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress;
import com.google.refine.importers.ImportingParserBase;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingUtilities;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
+import com.google.refine.model.Row;
import com.google.refine.util.JSONUtilities;
/**
@@ -210,8 +214,22 @@ abstract public class TreeImportingParserBase extends ImportingParserBase {
boolean trimStrings = JSONUtilities.getBoolean(options, "trimStrings", true);
boolean storeEmptyStrings = JSONUtilities.getBoolean(options, "storeEmptyStrings", false);
boolean guessCellValueTypes = JSONUtilities.getBoolean(options, "guessCellValueTypes", true);
-
- XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2, trimStrings,
- storeEmptyStrings,guessCellValueTypes);
+
+ // copied from TabularImportingParserBase
+ boolean includeFileSources = JSONUtilities.getBoolean(options, "includeFileSources", false);
+ String fileNameColumnName = "File";
+ if (includeFileSources) {
+ if (project.columnModel.getColumnByName(fileNameColumnName) == null) {
+ try {
+ project.columnModel.addColumn(
+ 0, new Column(project.columnModel.allocateNewCellIndex(), fileNameColumnName), false);
+ } catch (ModelException e) {
+ // Ignore: We already checked for duplicate name.
+ }
+ }
+ }
+
+ XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2,
+ new ImportParameters(trimStrings, storeEmptyStrings,guessCellValueTypes, includeFileSources,fileSource));
}
}
diff --git a/main/src/com/google/refine/importers/tree/XmlImportUtilities.java b/main/src/com/google/refine/importers/tree/XmlImportUtilities.java
index 90401100d..65b287c81 100644
--- a/main/src/com/google/refine/importers/tree/XmlImportUtilities.java
+++ b/main/src/com/google/refine/importers/tree/XmlImportUtilities.java
@@ -245,26 +245,13 @@ public class XmlImportUtilities extends TreeImportUtilities {
return null;
}
- @Deprecated
- static public void importTreeData(
- TreeReader parser,
- Project project,
- String[] recordPath,
- ImportColumnGroup rootColumnGroup,
- int limit
- ) {
- importTreeData(parser, project, recordPath, rootColumnGroup, limit,true,false,true);
- }
-
static public void importTreeData(
TreeReader parser,
Project project,
String[] recordPath,
ImportColumnGroup rootColumnGroup,
int limit,
- boolean trimStrings,
- boolean storeEmptyStrings,
- boolean guessDataType
+ ImportParameters parameters
) {
if (logger.isTraceEnabled()) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)");
@@ -273,7 +260,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
while (parser.hasNext()) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
- findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--,trimStrings,storeEmptyStrings,guessDataType);
+ findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--,parameters);
}
}
} catch (TreeReaderException e) {
@@ -282,18 +269,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
- @Deprecated
- static protected void findRecord(
- Project project,
- TreeReader parser,
- String[] recordPath,
- int pathIndex,
- ImportColumnGroup rootColumnGroup,
- int limit
- ) throws TreeReaderException {
- findRecord(project, parser, recordPath, pathIndex, rootColumnGroup, limit, true, false,true);
- }
-
/**
*
* @param project
@@ -310,9 +285,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
int pathIndex,
ImportColumnGroup rootColumnGroup,
int limit,
- boolean trimStrings,
- boolean storeEmptyStrings,
- boolean guessDataType
+ ImportParameters parameters
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("findRecord(Project, TreeReader, String[], int, ImportColumnGroup - path:"+Arrays.toString(recordPath));
@@ -331,7 +304,8 @@ public class XmlImportUtilities extends TreeImportUtilities {
while (parser.hasNext() && limit != 0) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
- findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--,trimStrings,storeEmptyStrings,guessDataType);
+ findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--,
+ parameters);
} else if (eventType == Token.EndEntity) {
break;
} else if (eventType == Token.Value) {
@@ -340,13 +314,13 @@ public class XmlImportUtilities extends TreeImportUtilities {
String desiredFieldName = recordPath[pathIndex + 1];
String currentFieldName = parser.getFieldName();
if (desiredFieldName.equals(currentFieldName)) {
- processFieldAsRecord(project, parser, rootColumnGroup,trimStrings,storeEmptyStrings,guessDataType);
+ processFieldAsRecord(project, parser, rootColumnGroup,parameters);
}
}
}
}
} else {
- processRecord(project, parser, rootColumnGroup, trimStrings, storeEmptyStrings, guessDataType);
+ processRecord(project, parser, rootColumnGroup, parameters);
}
} else {
skip(parser);
@@ -364,17 +338,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
- /**
- * @deprecated on 20120907 by tfmorris -use {@link #processRecord(Project, TreeReader, ImportColumnGroup, boolean, boolean, boolean)}
- */
- @Deprecated
- static protected void processRecord(
- Project project,
- TreeReader parser,
- ImportColumnGroup rootColumnGroup
- ) throws TreeReaderException {
- processRecord(project, parser, rootColumnGroup, true, false, true);
- }
/**
* processRecord parses Tree data for a single element and it's sub-elements,
@@ -388,32 +351,18 @@ public class XmlImportUtilities extends TreeImportUtilities {
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup,
- boolean trimStrings,
- boolean storeEmptyStrings,
- boolean guessDataType
+ ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processRecord(Project,TreeReader,ImportColumnGroup)");
}
ImportRecord record = new ImportRecord();
- processSubRecord(project, parser, rootColumnGroup, record, 0, trimStrings, storeEmptyStrings, guessDataType);
- addImportRecordToProject(record, project);
+ processSubRecord(project, parser, rootColumnGroup, record, 0, parameter);
+ addImportRecordToProject(record, project, parameter.includeFileSources, parameter.fileSource);
}
- /**
- * @deprecated 20120907 by tfmorris - use {@link #processFieldAsRecord(Project, TreeReader, ImportColumnGroup, boolean, boolean, boolean)}
- */
- @Deprecated
- static protected void processFieldAsRecord(
- Project project,
- TreeReader parser,
- ImportColumnGroup rootColumnGroup
- ) throws TreeReaderException {
- processFieldAsRecord(project, parser, rootColumnGroup, true, false, true);
- }
-
-
+
/**
* processFieldAsRecord parses Tree data for a single element and it's sub-elements,
* adding the parsed data as a row to the project
@@ -426,9 +375,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
Project project,
TreeReader parser,
ImportColumnGroup rootColumnGroup,
- boolean trimStrings,
- boolean storeEmptyStrings,
- boolean guessDataType
+ ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processFieldAsRecord(Project,TreeReader,ImportColumnGroup)");
@@ -437,10 +384,10 @@ public class XmlImportUtilities extends TreeImportUtilities {
ImportRecord record = null;
if (value instanceof String) {
String text = (String) value;
- if (trimStrings) {
+ if (parameter.trimStrings) {
text = text.trim();
}
- if (text.length() > 0 | !storeEmptyStrings) {
+ if (text.length() > 0 | !parameter.storeEmptyStrings) {
record = new ImportRecord();
addCell(
project,
@@ -448,8 +395,8 @@ public class XmlImportUtilities extends TreeImportUtilities {
record,
parser.getFieldName(),
(String) value,
- storeEmptyStrings,
- guessDataType
+ parameter.storeEmptyStrings,
+ parameter.guessDataType
);
}
} else {
@@ -463,20 +410,25 @@ public class XmlImportUtilities extends TreeImportUtilities {
);
}
if (record != null) {
- addImportRecordToProject(record, project);
+ addImportRecordToProject(record, project,
+ parameter.includeFileSources, parameter.fileSource);
}
}
- static protected void addImportRecordToProject(ImportRecord record, Project project) {
+ static protected void addImportRecordToProject(ImportRecord record, Project project,
+ boolean includeFileSources, String fileSource) {
for (List row : record.rows) {
if (row.size() > 0) {
- Row realRow = null;
+ Row realRow = new Row(row.size()); ;
for (int c = 0; c < row.size(); c++) {
+ if (c == 0 && includeFileSources) { // to add the file source:
+ realRow.setCell(
+ 0,
+ new Cell(fileSource, null));
+ continue;
+ }
Cell cell = row.get(c);
if (cell != null) {
- if (realRow == null) {
- realRow = new Row(row.size());
- }
realRow.setCell(c, cell);
}
}
@@ -486,19 +438,6 @@ public class XmlImportUtilities extends TreeImportUtilities {
}
}
}
-
- /**
- * @deprecated by tfmorris use {@link #processSubRecord(Project, TreeReader, ImportColumnGroup, ImportRecord, int, boolean, boolean, boolean)}
- */
- @Deprecated
- static protected void processSubRecord( Project project,
- TreeReader parser,
- ImportColumnGroup columnGroup,
- ImportRecord record,
- int level
- ) throws TreeReaderException {
- processSubRecord(project, parser, columnGroup, record, level, true, false, true);
- }
/**
*
@@ -514,9 +453,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
ImportColumnGroup columnGroup,
ImportRecord record,
int level,
- boolean trimStrings,
- boolean storeEmptyStrings,
- boolean guessDataType
+ ImportParameters parameter
) throws TreeReaderException {
if (logger.isTraceEnabled()) {
logger.trace("processSubRecord(Project,TreeReader,ImportColumnGroup,ImportRecord) lvl:"+level+" "+columnGroup);
@@ -536,18 +473,18 @@ public class XmlImportUtilities extends TreeImportUtilities {
int attributeCount = parser.getAttributeCount();
for (int i = 0; i < attributeCount; i++) {
String text = parser.getAttributeValue(i);
- if (trimStrings) {
+ if (parameter.trimStrings) {
text = text.trim();
}
- if (text.length() > 0 | !storeEmptyStrings) {
+ if (text.length() > 0 | !parameter.storeEmptyStrings) {
addCell(
project,
thisColumnGroup,
record,
composeName(parser.getAttributePrefix(i), parser.getAttributeLocalName(i)),
text,
- storeEmptyStrings,
- guessDataType
+ parameter.storeEmptyStrings,
+ parameter.guessDataType
);
}
}
@@ -561,9 +498,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
thisColumnGroup,
record,
level+1,
- trimStrings,
- storeEmptyStrings,
- guessDataType
+ parameter
);
} else if (//eventType == XMLStreamConstants.CDATA ||
eventType == Token.Value) { //XMLStreamConstants.CHARACTERS) {
@@ -572,7 +507,7 @@ public class XmlImportUtilities extends TreeImportUtilities {
if (value instanceof String) {
String text = (String) value;
addCell(project, thisColumnGroup, record, colName, text,
- storeEmptyStrings, guessDataType);
+ parameter.storeEmptyStrings, parameter.guessDataType);
} else {
addCell(project, thisColumnGroup, record, colName, value);
}
diff --git a/main/tests/server/src/com/google/refine/tests/exporters/HtmlExporterTests.java b/main/tests/server/src/com/google/refine/tests/exporters/HtmlExporterTests.java
index bf3fbd50d..47b38ca0e 100644
--- a/main/tests/server/src/com/google/refine/tests/exporters/HtmlExporterTests.java
+++ b/main/tests/server/src/com/google/refine/tests/exporters/HtmlExporterTests.java
@@ -117,7 +117,9 @@ public class HtmlExporterTests extends RefineTest {
}
Assert.assertEquals(writer.toString(), "\n" +
- "" + TEST_PROJECT_NAME + " \n" +
+ "\n" + "" + TEST_PROJECT_NAME + " \n" +
+ "\n" +
"column0 column1 " + TEST_PROJECT_NAME + " \n" +
+ "\n" + "" + TEST_PROJECT_NAME + " \n" +
+ "\n" +
"row0cell0 row0cell1 " + TEST_PROJECT_NAME + " \n" +
+ "\n" + "" + TEST_PROJECT_NAME + " \n" +
+ "\n" +
"column0 column1 column2 |