diff --git a/extensions/wikidata/.classpath b/extensions/wikidata/.classpath index 0f8fa0f27..d44b29c4b 100644 --- a/extensions/wikidata/.classpath +++ b/extensions/wikidata/.classpath @@ -19,5 +19,6 @@ + diff --git a/extensions/wikidata/module/MOD-INF/controller.js b/extensions/wikidata/module/MOD-INF/controller.js index 85aece3db..3ea39a2f5 100644 --- a/extensions/wikidata/module/MOD-INF/controller.js +++ b/extensions/wikidata/module/MOD-INF/controller.js @@ -35,6 +35,8 @@ function init() { */ Packages.com.google.refine.operations.OperationRegistry.registerOperation( module, "save-wikibase-schema", Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation); + Packages.com.google.refine.operations.OperationRegistry.registerOperation( + module, "perform-wikibase-edits", Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation); /* * Exporters @@ -49,6 +51,8 @@ function init() { */ RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand()); RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand()); + RefineServlet.registerCommand(module, "perform-wikibase-edits", new PerformWikibaseEditsCommand()); + RefineServlet.registerCommand(module, "login", new LoginCommand()); /* * Resources @@ -59,6 +63,8 @@ function init() { [ "scripts/menu-bar-extension.js", "scripts/dialogs/schema-alignment-dialog.js", + "scripts/dialogs/manage-account-dialog.js", + "scripts/dialogs/perform-edits-dialog.js", ]); ClientSideResourceManager.addPaths( @@ -66,76 +72,8 @@ function init() { module, [ "styles/dialogs/schema-alignment-dialog.less", + "styles/dialogs/manage-account-dialog.less", ]); } -function process(path, request, response) { - // Analyze path and handle this request yourself. - /* - var loggerFactory = Packages.org.slf4j.LoggerFactory; - var logger = loggerFactory.getLogger("rdf_extension"); - var method = request.getMethod(); - - logger.info('receiving request for ' + path); - if(rdfReconcileExtension.isKnownRequestUrl(path)){ - var command = rdfReconcileExtension.getCommand(path, request); - logger.info('command is ' + command); - var serviceName = rdfReconcileExtension.getServiceName(path); - logger.info('command is ' + command + ', while service name is ' + serviceName); - if(command && command !== 'unknown'){ - var jsonResponse; - if(command==='metadata'){ - jsonResponse = GRefineServiceManager.singleton.metadata(serviceName,request); - }else if(command==='multi-reconcile'){ - jsonResponse = GRefineServiceManager.singleton.multiReconcile(serviceName,request); - }else if (command==='suggest-type'){ - jsonResponse = GRefineServiceManager.singleton.suggestType(serviceName,request); - }else if (command==='flyout-type'){ - jsonResponse = GRefineServiceManager.singleton.previewType(serviceName,request); - }else if (command==='suggest-property'){ - jsonResponse = GRefineServiceManager.singleton.suggestProperty(serviceName,request); - }else if (command==='flyout-property'){ - jsonResponse = GRefineServiceManager.singleton.previewProperty(serviceName,request); - }else if (command==='suggest-entity'){ - jsonResponse = GRefineServiceManager.singleton.suggestEntity(serviceName,request); - }else if (command==='flyout-entity'){ - jsonResponse = GRefineServiceManager.singleton.previewEntity(serviceName,request); - }else if (command==='preview-resource-template'){ - var htmlResponse = GRefineServiceManager.singleton.getHtmlOfResourcePreviewTemplate(serviceName,request); - if(htmlResponse){ - butterfly.sendString(request, response, htmlResponse ,"UTF-8", "text/html"); - }else{ - butterfly.sendError(request, response, 404, "unknown service"); - } - return; - }else if (command==='view-resource'){ - var id = request.getParameter('id'); - butterfly.redirect(request,response,id); - return; - }else if (command ==='preview-resource'){ - logger.info("id is " + request.getParameter("id")); - var htmlResponse = GRefineServiceManager.singleton.previewResource(serviceName,request); - if(htmlResponse){ - butterfly.sendString(request, response, htmlResponse ,"UTF-8", "text/html"); - }else{ - butterfly.sendError(request, response, 404, "unknown service"); - } - return; - } - - if(jsonResponse){ - logger.info(jsonResponse); - butterfly.sendString(request, response, jsonResponse ,"UTF-8", "text/javascript"); - return; - }else{ - butterfly.sendError(request, response, 404, "unknown service"); - } - } - //else it is an unknown command... do nothing - } - - if (path == "/" || path == "") { - butterfly.redirect(request, response, "index.html"); - } */ -} diff --git a/extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar b/extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar new file mode 100644 index 000000000..e6c520e9b Binary files /dev/null and b/extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar differ diff --git a/extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar b/extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar new file mode 100644 index 000000000..e1263151d Binary files /dev/null and b/extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar differ diff --git a/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.html b/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.html new file mode 100644 index 000000000..a63255bb3 --- /dev/null +++ b/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.html @@ -0,0 +1,42 @@ +
+
Wikidata account
+
+

+ Logging in to Wikidata will allow you to perform edits directly from OpenRefine. + Your credentials will be stored unencrypted in OpenRefine's preferences. +

+
+ +
+

You are logged in as: +

+
+ + +
+ +
+
+
+
diff --git a/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.js b/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.js new file mode 100644 index 000000000..f2d0e2b53 --- /dev/null +++ b/extensions/wikidata/module/scripts/dialogs/manage-account-dialog.js @@ -0,0 +1,75 @@ +var ManageAccountDialog = {}; +ManageAccountDialog.launch = function(logged_in_username, callback) { + var self = this; + var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/manage-account-dialog.html")); + var elmts = this._elmts = DOM.bind(frame); + + this._level = DialogSystem.showDialog(frame); + + var dismiss = function() { + DialogSystem.dismissUntil(self._level - 1); + }; + + if (logged_in_username != null) { + elmts.loginArea.hide(); + } else { + elmts.logoutArea.hide(); + } + + elmts.loggedInUsername.text(logged_in_username); + + frame.find('.cancel-button').click(function() { + dismiss(); + callback(null); + }); + + elmts.loginButton.click(function() { + $.post( + "command/wikidata/login", + elmts.loginForm.serialize(), + function(data) { + if (data.logged_in) { + dismiss(); + callback(data.username); + } else { + elmts.invalidCredentials.text("Invalid credentials."); + } + }); + }); + + elmts.logoutButton.click(function() { + $.post( + "command/wikidata/login", + "logout=true", + function(data) { + if (!data.logged_in) { + dismiss(); + callback(null); + } + }); + }); +}; + +ManageAccountDialog.isLoggedIn = function(callback) { + $.get( + "command/wikidata/login", + function(data) { + callback(data.username); + }); +}; + +ManageAccountDialog.ensureLoggedIn = function(callback) { + ManageAccountDialog.isLoggedIn(function(logged_in_username) { + if (logged_in_username == null) { + ManageAccountDialog.launch(null, callback); + } else { + callback(logged_in_username); + } + }); +}; + +ManageAccountDialog.checkAndLaunch = function () { + ManageAccountDialog.isLoggedIn(function(logged_in_username) { + ManageAccountDialog.launch(logged_in_username, function(success) { }); + }); +}; diff --git a/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.html b/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.html new file mode 100644 index 000000000..3adb0f596 --- /dev/null +++ b/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.html @@ -0,0 +1,21 @@ +
+
Peform edits on Wikidata
+
+

+ Please review your edits before pushing them to Wikidata. + Consider requesting feedback at the Data Import Hub + for large datasets. +

+
+

You are logged in as .

+
+ + +
+ +
+
+
diff --git a/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.js b/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.js new file mode 100644 index 000000000..65d481cfa --- /dev/null +++ b/extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.js @@ -0,0 +1,41 @@ +var PerformEditsDialog = {}; + +PerformEditsDialog.launch = function(logged_in_username) { + var self = this; + var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/perform-edits-dialog.html")); + var elmts = this._elmts = DOM.bind(frame); + + this._level = DialogSystem.showDialog(frame); + + var dismiss = function() { + DialogSystem.dismissUntil(self._level - 1); + }; + + elmts.loggedInUsername.text(logged_in_username); + + frame.find('.cancel-button').click(function() { + dismiss(); + }); + + elmts.performEditsButton.click(function() { + Refine.postProcess( + "wikidata", + "perform-wikibase-edits", + {}, + elmts.performEditsForm.serialize(), + {}, + { onDone: + function() { + dismiss(); + } + }); + }); +}; + +PerformEditsDialog.checkAndLaunch = function () { + ManageAccountDialog.ensureLoggedIn(function(logged_in_username) { + if (logged_in_username) { + PerformEditsDialog.launch(logged_in_username); + } + }); +}; diff --git a/extensions/wikidata/module/scripts/menu-bar-extension.js b/extensions/wikidata/module/scripts/menu-bar-extension.js index dba8b5482..e88160b1e 100644 --- a/extensions/wikidata/module/scripts/menu-bar-extension.js +++ b/extensions/wikidata/module/scripts/menu-bar-extension.js @@ -1,67 +1,3 @@ -/* -ExporterManager.MenuItems.push({});//add separator -ExporterManager.MenuItems.push( - { - "id" : "exportRdfXml", - "label":"RDF as RDF/XML", - "click": function() { RdfExporterMenuBar.exportRDF("rdf", "rdf");} - } -); -ExporterManager.MenuItems.push( - { - "id" : "exportRdfTurtle", - "label":"RDF as Turtle", - "click": function() { RdfExporterMenuBar.exportRDF("Turtle", "ttl"); } - } -); - -RdfExporterMenuBar = {}; - -RdfExporterMenuBar.exportRDF = function(format, ext) { - if (!theProject.overlayModels.rdfSchema) { - alert( - "You haven't done any RDF schema alignment yet!" - ); - } else { - RdfExporterMenuBar.rdfExportRows(format, ext); - } -}; - -RdfExporterMenuBar.rdfExportRows = function(format, ext) { - var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); - var form = document.createElement("form"); - $(form) - .css("display", "none") - .attr("method", "post") - .attr("action", "command/core/export-rows/" + name + "." + ext) - .attr("target", "gridworks-export"); - - $('') - .attr("name", "engine") - .attr("value", JSON.stringify(ui.browsingEngine.getJSON())) - .appendTo(form); - $('') - .attr("name", "project") - .attr("value", theProject.id) - .appendTo(form); - $('') - .attr("name", "format") - .attr("value", format) - .appendTo(form); - - document.body.appendChild(form); - - window.open("about:blank", "gridworks-export"); - form.submit(); - - document.body.removeChild(form); -}; - -RdfExporterMenuBar.editRdfSchema = function(reset) { - new RdfSchemaAlignmentDialog(reset ? null : theProject.overlayModels.rdfSchema); -}; -*/ - ExporterManager.MenuItems.push({}); ExporterManager.MenuItems.push( { @@ -109,15 +45,26 @@ $(function(){ "label": "Wikidata", "submenu" : [ { - "id": "wikidata/edit-schema", + id: "wikidata/edit-schema", label: "Edit Wikibase schema...", click: function() { SchemaAlignmentDialog.launch(false); } }, + { + id:"wikidata/manage-account", + label: "Manage account", + click: function() { ManageAccountDialog.checkAndLaunch(); } + }, + { + id:"wikidata/perform-edits", + label: "Push to Wikidata...", + click: function() { PerformEditsDialog.checkAndLaunch(); } + }, { - "id":"wikidata/export-qs", - "label": "Export to QuickStatements", - "click": function() { WikibaseExporterMenuBar.exportTo("quickstatements"); } - } + id:"wikidata/export-qs", + label: "Export to QuickStatements", + click: function() { WikibaseExporterMenuBar.exportTo("quickstatements"); } + }, + ] } ); diff --git a/extensions/wikidata/module/styles/dialogs/manage-account-dialog.less b/extensions/wikidata/module/styles/dialogs/manage-account-dialog.less new file mode 100644 index 000000000..f57cad152 --- /dev/null +++ b/extensions/wikidata/module/styles/dialogs/manage-account-dialog.less @@ -0,0 +1,43 @@ +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +@import-less url("../theme.less"); + +.wikibase-login-form { + text-align: center; +} + +.wikibase-login-buttons { + text-align: right; +} + diff --git a/extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java b/extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java new file mode 100644 index 000000000..83ccea2a1 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java @@ -0,0 +1,53 @@ +package org.openrefine.wikidata.commands; + +import java.io.IOException; +import java.io.StringWriter; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONWriter; +import org.openrefine.wikidata.editing.ConnectionManager; + +import com.google.refine.commands.Command; + + +public class LoginCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String username = request.getParameter("wb-username"); + String password = request.getParameter("wb-password"); + ConnectionManager manager = ConnectionManager.getInstance(); + if (username != null && password != null) { + manager.login(username, password); + } else if ("true".equals(request.getParameter("logout"))) { + manager.logout(); + } + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + StringWriter sb = new StringWriter(2048); + JSONWriter writer = new JSONWriter(sb, 32); + + try { + writer.object(); + writer.key("logged_in"); + writer.value(manager.isLoggedIn()); + writer.key("username"); + writer.value(manager.getUsername()); + writer.endObject(); + } catch (JSONException e) { + e.printStackTrace(); + } + respond(response, sb.toString()); + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doPost(request, response); + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java b/extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java new file mode 100644 index 000000000..49dfc1453 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java @@ -0,0 +1,26 @@ +package org.openrefine.wikidata.commands; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; +import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation; +import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation.DuplicateDetectionStrategy; +import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation.OnDuplicateAction; + +import com.google.refine.commands.EngineDependentCommand; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; + +public class PerformWikibaseEditsCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig) + throws Exception { + String strategy = request.getParameter("strategy"); + String action = request.getParameter("action"); + return new PerformWikibaseEditsOperation(engineConfig, + DuplicateDetectionStrategy.valueOf(strategy), + OnDuplicateAction.valueOf(action)); + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java b/extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java new file mode 100644 index 000000000..51b638882 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java @@ -0,0 +1,112 @@ +package org.openrefine.wikidata.editing; + +import java.io.IOException; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikidata.wdtk.wikibaseapi.LoginFailedException; + +import com.google.refine.ProjectManager; +import com.google.refine.preference.PreferenceStore; + + + +/** + * Manages a connection to Wikidata, with login credentials stored + * in the preferences. + * + * Ideally, we should store only the cookies and not the password. + * But Wikidata-Toolkit does not allow for that as cookies are kept + * private. + * + * This class is also hard-coded for Wikidata: generalization to other + * Wikibase instances should be feasible though. + * + * @author antonin + */ + +public class ConnectionManager { + public static final String PREFERENCE_STORE_KEY = "wikidata_credentials"; + + private PreferenceStore prefStore; + private ApiConnection connection; + + private static class ConnectionManagerHolder { + private static final ConnectionManager instance = new ConnectionManager(); + } + + public static ConnectionManager getInstance() { + return ConnectionManagerHolder.instance; + } + + private ConnectionManager() { + prefStore = ProjectManager.singleton.getPreferenceStore(); + connection = null; + } + + public void login(String username, String password) { + try { + JSONArray array = new JSONArray(); + JSONObject obj = new JSONObject(); + obj.put("username", username); + obj.put("password", password); + array.put(obj); + prefStore.put(PREFERENCE_STORE_KEY, array); + } catch (JSONException e) { + e.printStackTrace(); + } + connection = ApiConnection.getWikidataApiConnection(); + try { + connection.login(username, password); + } catch (LoginFailedException e) { + connection = null; + } + } + + public void restorePreviousLogin() { + JSONArray array = (JSONArray) prefStore.get(PREFERENCE_STORE_KEY); + if (array.length() > 0) { + JSONObject obj; + try { + obj = array.getJSONObject(0); + String username = obj.getString("username"); + String password = obj.getString("password"); + login(username, password); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } + + public void logout() { + prefStore.put(PREFERENCE_STORE_KEY, new JSONArray()); + if (connection != null) { + try { + connection.logout(); + connection = null; + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public ApiConnection getConnection() { + return connection; + } + + public boolean isLoggedIn() { + return connection != null; + } + + public String getUsername() { + if (connection != null) { + return connection.getCurrentUser(); + } else { + return null; + } + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java b/extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java new file mode 100644 index 000000000..180492667 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java @@ -0,0 +1,65 @@ +package org.openrefine.wikidata.editing; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.Jsonizable; + +/** + * This is just the necessary bits to store Wikidata credentials + * in OpenRefine's preference store. + * + * @author antonin + * + */ +class WikibaseCredentials implements Jsonizable { + + private String username; + private String password; + + public WikibaseCredentials() { + username = null; + password = null; + } + + public WikibaseCredentials(String username, String password) { + this.username = username; + this.password = password; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public boolean isNonNull() { + return username != null && password != null && ! "null".equals(username) && ! "null".equals(password); + } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("class"); + writer.value(this.getClass().getName()); + writer.key("username"); + writer.value(username); + writer.key("password"); + writer.value(password); + writer.endObject(); + } + + public static WikibaseCredentials load(JSONObject obj) throws JSONException { + return new WikibaseCredentials( + obj.getString("username"), + obj.getString("password")); + } + +} + diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index 52f90c536..3d163a424 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -9,6 +9,7 @@ import com.google.refine.browsing.Engine; import com.google.refine.exporters.WriterExporter; import com.google.refine.model.Project; +import org.openrefine.wikidata.schema.ItemUpdate; import org.openrefine.wikidata.schema.WikibaseSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +17,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim; import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; -import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; @@ -24,7 +24,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Reference; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; -import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import org.wikidata.wdtk.datamodel.interfaces.StringValue; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; import org.wikidata.wdtk.datamodel.interfaces.Value; @@ -54,29 +53,25 @@ public class QuickStatementsExporter implements WriterExporter { } public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer) throws IOException { - List items = schema.evaluate(project, engine); - for (ItemDocument item : items) { + List items = schema.evaluate(project, engine); + for (ItemUpdate item : items) { translateItem(item, writer); } } - protected void translateItem(ItemDocument item, Writer writer) throws IOException { + protected void translateItem(ItemUpdate item, Writer writer) throws IOException { if (item.getItemId().equals(ItemIdValue.NULL)) { writer.write("CREATE\n"); } - for (StatementGroup group : item.getStatementGroups()) { - translateStatementGroup(group, writer); + for (Statement s : item.getAddedStatements()) { + translateStatement(s, s.getClaim().getMainSnak().getPropertyId().getId(), true, writer); + } + for (Statement s : item.getDeletedStatements()) { + translateStatement(s, s.getClaim().getMainSnak().getPropertyId().getId(), false, writer); } } - protected void translateStatementGroup(StatementGroup group, Writer writer) throws IOException { - String pid = group.getProperty().getId(); - for(Statement statement : group.getStatements()) { - translateStatement(statement, pid, writer); - } - } - - protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException { + protected void translateStatement(Statement statement, String pid, boolean add, Writer writer) throws IOException { Claim claim = statement.getClaim(); String qid = claim.getSubject().getId(); if (claim.getSubject().equals(ItemIdValue.NULL)) { @@ -86,6 +81,9 @@ public class QuickStatementsExporter implements WriterExporter { ValueVisitor vv = new ValuePrinter(); String targetValue = val.accept(vv); if (targetValue != null) { + if (! add) { + writer.write("- "); + } writer.write(qid + "\t" + pid + "\t" + targetValue); for(SnakGroup q : claim.getQualifiers()) { translateSnakGroup(q, false, writer); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java new file mode 100644 index 000000000..f2343eea5 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java @@ -0,0 +1,242 @@ +package org.openrefine.wikidata.operations; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import com.google.common.collect.Lists; + +import org.openrefine.wikidata.editing.ConnectionManager; +import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation.WikibaseSchemaChange; +import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.schema.WikibaseSchema; +import org.wikidata.wdtk.datamodel.helpers.EntityDocumentBuilder; +import org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder; +import org.wikidata.wdtk.datamodel.interfaces.EntityDocument; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.util.WebResourceFetcherImpl; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikidata.wdtk.wikibaseapi.LoginFailedException; +import org.wikidata.wdtk.wikibaseapi.StatementUpdate; +import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; +import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; +import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; +import org.wikidata.wdtk.datamodel.interfaces.Statement; + +import com.google.refine.browsing.Engine; +import com.google.refine.history.Change; +import com.google.refine.history.HistoryEntry; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.model.changes.ReconChange; +import com.google.refine.operations.EngineDependentOperation; +import com.google.refine.operations.OperationRegistry; +import com.google.refine.operations.recon.ReconOperation; +import com.google.refine.process.LongRunningProcess; +import com.google.refine.process.Process; +import com.google.refine.util.ParsingUtilities; +import com.google.refine.util.Pool; + + +public class PerformWikibaseEditsOperation extends EngineDependentOperation { + public enum DuplicateDetectionStrategy { + PROPERTY, SNAK, SNAK_QUALIFIERS + } + + public enum OnDuplicateAction { + SKIP, MERGE + } + + private DuplicateDetectionStrategy strategy; + private OnDuplicateAction duplicateAction; + + public PerformWikibaseEditsOperation( + JSONObject engineConfig, + DuplicateDetectionStrategy strategy, + OnDuplicateAction duplicateAction) { + super(engineConfig); + this.strategy = strategy; + this.duplicateAction = duplicateAction; + + // getEngine(request, project); + } + + static public AbstractOperation reconstruct(Project project, JSONObject obj) + throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + String strategy = obj.getString("duplicate_strategy"); + String action = obj.getString("duplicate_action"); + return new PerformWikibaseEditsOperation( + engineConfig, + DuplicateDetectionStrategy.valueOf(strategy), + OnDuplicateAction.valueOf(action)); + } + + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("op"); + writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); + writer.value("Perform Wikibase edits"); + writer.key("duplicate_strategy"); + writer.value(strategy.name()); + writer.key("duplicate_action"); + writer.value(duplicateAction.name()); + writer.endObject(); + } + + @Override + protected String getBriefDescription(Project project) { + return "Peform edits on Wikidata"; + } + + @Override + public Process createProcess(Project project, Properties options) throws Exception { + return new PerformEditsProcess( + project, + createEngine(project), + getBriefDescription(project), + "#openrefine" + ); + } + static public class PerformWikibaseEditsChange implements Change { + + @Override + public void apply(Project project) { + // this does not do anything to the project (we don't re-run changes on Wikidata) + } + + @Override + public void revert(Project project) { + // this does not do anything (we don't revert changes on Wikidata either) + } + + @Override + public void save(Writer writer, Properties options) + throws IOException { + writer.write("/ec/\n"); // end of change + } + + static public Change load(LineNumberReader reader, Pool pool) + throws Exception { + return new PerformWikibaseEditsChange(); + } + + } + + public class PerformEditsProcess extends LongRunningProcess implements Runnable { + + protected Project _project; + protected Engine _engine; + protected WikibaseSchema _schema; + protected String _summary; + protected final long _historyEntryID; + + protected PerformEditsProcess(Project project, + Engine engine, String description, String summary) { + super(description); + this._project = project; + this._engine = engine; + this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema"); + this._summary = summary; + this._historyEntryID = HistoryEntry.allocateID(); + } + + @Override + public void run() { + // TODO Auto-generated method stub + + WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension"); + ConnectionManager manager = ConnectionManager.getInstance(); + if (!manager.isLoggedIn()) { + return; + } + ApiConnection connection = manager.getConnection(); + + //WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, schema.getBaseUri()); + WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseUri()); + //wbde.disableEditing(); + + // Evaluate the schema + List itemDocuments = _schema.evaluate(_project, _engine); + + // Group statements by item + Map updates = new HashMap(); + for(ItemUpdate update : itemDocuments) { + if (update.isNull()) { + continue; + } + + ItemIdValue qid = update.getItemId(); + if (updates.containsKey(qid)) { + ItemUpdate oldUpdate = updates.get(qid); + oldUpdate.merge(update); + } else { + updates.put(qid, update); + } + } + + /** + * TODO: + * - support for new items + * - support for duplicate strategy and action + */ + + // Perform edits + int totalItemUpdates = updates.size(); + int updatesDone = 0; + for(ItemUpdate update : updates.values()) { + try { + wbde.updateStatements(update.getItemId(), update.getAddedStatements(), update.getDeletedStatements(), _summary); + + } catch (MediaWikiApiErrorException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + updatesDone++; + _progress = (100*updatesDone) / totalItemUpdates; + + if(_canceled) { + break; + } + } + _progress = 100; + + if (!_canceled) { + Change change = new PerformWikibaseEditsChange(); + + HistoryEntry historyEntry = new HistoryEntry( + _historyEntryID, + _project, + _description, + PerformWikibaseEditsOperation.this, + change + ); + + _project.history.addEntry(historyEntry); + _project.processManager.onDoneProcess(this); + } + } + + @Override + protected Runnable getRunnable() { + return this; + } + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java new file mode 100644 index 000000000..ac8c40a60 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java @@ -0,0 +1,68 @@ +package org.openrefine.wikidata.schema; + +import java.util.ArrayList; +import java.util.List; + +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; + + +/** + * A class to plan an update of an item, after evaluating the statements + * but before fetching the current content of the item (this is why it does not + * extend StatementsUpdate). + * + * @author antonin + */ +public class ItemUpdate { + private ItemIdValue qid; + private List addedStatements; + private List deletedStatements; + + public ItemUpdate(ItemIdValue qid) { + this.qid = qid; + this.addedStatements = new ArrayList(); + this.deletedStatements = new ArrayList(); + } + + public void addStatement(Statement s) { + addedStatements.add(s); + } + + public void deleteStatement(Statement s) { + deletedStatements.add(s); + } + + public void addStatements(List l) { + addedStatements.addAll(l); + } + + public void deleteStatements(List l) { + deletedStatements.addAll(l); + } + + public ItemIdValue getItemId() { + return qid; + } + + public List getAddedStatements() { + return addedStatements; + } + + public List getDeletedStatements() { + return deletedStatements; + } + + /** + * Merges all the changes in other into this instance. + * @param other: the other change that should be merged + */ + public void merge(ItemUpdate other) { + addStatements(other.getAddedStatements()); + deleteStatements(other.getDeletedStatements()); + } + + public boolean isNull() { + return addedStatements.isEmpty() && deletedStatements.isEmpty(); + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java index 80df0151a..e01f86526 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java @@ -52,15 +52,15 @@ public class WbItemDocumentExpr extends BiJsonizable { statementExprs); } - public ItemDocument evaluate(ExpressionContext ctxt) throws SkipStatementException { + public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipStatementException { ItemIdValue subjectId = subjectExpr.evaluate(ctxt); - ItemDocumentBuilder builder = ItemDocumentBuilder.forItemId(subjectId); + ItemUpdate update = new ItemUpdate(subjectId); for(WbStatementGroupExpr expr : statementGroupExprs) { for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) { - builder.withStatement(s); + update.addStatement(s); } } - return builder.build(); + return update; } public String getJsonType() { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 5dbe41ca7..4d6f2eea5 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -58,8 +58,8 @@ public class WikibaseSchema implements OverlayModel { return itemDocumentExprs; } - public List evaluate(ExpressionContext ctxt) { - List result = new ArrayList(); + public List evaluate(ExpressionContext ctxt) { + List result = new ArrayList(); for (WbItemDocumentExpr expr : itemDocumentExprs) { try { @@ -71,16 +71,16 @@ public class WikibaseSchema implements OverlayModel { return result; } - public List evaluate(Project project, Engine engine) { - List result = new ArrayList(); + public List evaluate(Project project, Engine engine) { + List result = new ArrayList(); FilteredRows filteredRows = engine.getAllFilteredRows(); filteredRows.accept(project, new EvaluatingRowVisitor(result)); return result; } protected class EvaluatingRowVisitor implements RowVisitor { - private List result; - public EvaluatingRowVisitor(List result) { + private List result; + public EvaluatingRowVisitor(List result) { this.result = result; }