Support for Wikidata editing from OpenRefine
This commit is contained in:
Antonin Delpeuch
parent
67e5bcd504
commit
8f4d998e21
@ -19,5 +19,6 @@
|
||||
</classpathentry>
|
||||
<classpathentry combineaccessrules="false" kind="src" path="/OpenRefine"/>
|
||||
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-datamodel-0.8.0-SNAPSHOT-jar-with-dependencies.jar" sourcepath="module/MOD-INF/lib-src/wdtk-datamodel-0.8.0-SNAPSHOT-sources.jar"/>
|
||||
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar" sourcepath="module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar"/>
|
||||
<classpathentry kind="output" path="module/MOD-INF/classes"/>
|
||||
</classpath>
|
||||
|
||||
@ -35,6 +35,8 @@ function init() {
|
||||
*/
|
||||
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
|
||||
module, "save-wikibase-schema", Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation);
|
||||
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
|
||||
module, "perform-wikibase-edits", Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation);
|
||||
|
||||
/*
|
||||
* Exporters
|
||||
@ -49,6 +51,8 @@ function init() {
|
||||
*/
|
||||
RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand());
|
||||
RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand());
|
||||
RefineServlet.registerCommand(module, "perform-wikibase-edits", new PerformWikibaseEditsCommand());
|
||||
RefineServlet.registerCommand(module, "login", new LoginCommand());
|
||||
|
||||
/*
|
||||
* Resources
|
||||
@ -59,6 +63,8 @@ function init() {
|
||||
[
|
||||
"scripts/menu-bar-extension.js",
|
||||
"scripts/dialogs/schema-alignment-dialog.js",
|
||||
"scripts/dialogs/manage-account-dialog.js",
|
||||
"scripts/dialogs/perform-edits-dialog.js",
|
||||
]);
|
||||
|
||||
ClientSideResourceManager.addPaths(
|
||||
@ -66,76 +72,8 @@ function init() {
|
||||
module,
|
||||
[
|
||||
"styles/dialogs/schema-alignment-dialog.less",
|
||||
"styles/dialogs/manage-account-dialog.less",
|
||||
]);
|
||||
|
||||
}
|
||||
|
||||
function process(path, request, response) {
|
||||
// Analyze path and handle this request yourself.
|
||||
/*
|
||||
var loggerFactory = Packages.org.slf4j.LoggerFactory;
|
||||
var logger = loggerFactory.getLogger("rdf_extension");
|
||||
var method = request.getMethod();
|
||||
|
||||
logger.info('receiving request for ' + path);
|
||||
if(rdfReconcileExtension.isKnownRequestUrl(path)){
|
||||
var command = rdfReconcileExtension.getCommand(path, request);
|
||||
logger.info('command is ' + command);
|
||||
var serviceName = rdfReconcileExtension.getServiceName(path);
|
||||
logger.info('command is ' + command + ', while service name is ' + serviceName);
|
||||
if(command && command !== 'unknown'){
|
||||
var jsonResponse;
|
||||
if(command==='metadata'){
|
||||
jsonResponse = GRefineServiceManager.singleton.metadata(serviceName,request);
|
||||
}else if(command==='multi-reconcile'){
|
||||
jsonResponse = GRefineServiceManager.singleton.multiReconcile(serviceName,request);
|
||||
}else if (command==='suggest-type'){
|
||||
jsonResponse = GRefineServiceManager.singleton.suggestType(serviceName,request);
|
||||
}else if (command==='flyout-type'){
|
||||
jsonResponse = GRefineServiceManager.singleton.previewType(serviceName,request);
|
||||
}else if (command==='suggest-property'){
|
||||
jsonResponse = GRefineServiceManager.singleton.suggestProperty(serviceName,request);
|
||||
}else if (command==='flyout-property'){
|
||||
jsonResponse = GRefineServiceManager.singleton.previewProperty(serviceName,request);
|
||||
}else if (command==='suggest-entity'){
|
||||
jsonResponse = GRefineServiceManager.singleton.suggestEntity(serviceName,request);
|
||||
}else if (command==='flyout-entity'){
|
||||
jsonResponse = GRefineServiceManager.singleton.previewEntity(serviceName,request);
|
||||
}else if (command==='preview-resource-template'){
|
||||
var htmlResponse = GRefineServiceManager.singleton.getHtmlOfResourcePreviewTemplate(serviceName,request);
|
||||
if(htmlResponse){
|
||||
butterfly.sendString(request, response, htmlResponse ,"UTF-8", "text/html");
|
||||
}else{
|
||||
butterfly.sendError(request, response, 404, "unknown service");
|
||||
}
|
||||
return;
|
||||
}else if (command==='view-resource'){
|
||||
var id = request.getParameter('id');
|
||||
butterfly.redirect(request,response,id);
|
||||
return;
|
||||
}else if (command ==='preview-resource'){
|
||||
logger.info("id is " + request.getParameter("id"));
|
||||
var htmlResponse = GRefineServiceManager.singleton.previewResource(serviceName,request);
|
||||
if(htmlResponse){
|
||||
butterfly.sendString(request, response, htmlResponse ,"UTF-8", "text/html");
|
||||
}else{
|
||||
butterfly.sendError(request, response, 404, "unknown service");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if(jsonResponse){
|
||||
logger.info(jsonResponse);
|
||||
butterfly.sendString(request, response, jsonResponse ,"UTF-8", "text/javascript");
|
||||
return;
|
||||
}else{
|
||||
butterfly.sendError(request, response, 404, "unknown service");
|
||||
}
|
||||
}
|
||||
//else it is an unknown command... do nothing
|
||||
}
|
||||
|
||||
if (path == "/" || path == "") {
|
||||
butterfly.redirect(request, response, "index.html");
|
||||
} */
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,42 @@
|
||||
<div class="dialog-frame" style="width: 800px;">
|
||||
<div class="dialog-header" bind="dialogHeader">Wikidata account</div>
|
||||
<div class="dialog-body" bind="dialogBody">
|
||||
<p class="body-text">
|
||||
Logging in to Wikidata will allow you to perform edits directly from OpenRefine.
|
||||
Your credentials will be stored unencrypted in OpenRefine's preferences.
|
||||
</p>
|
||||
<div class="wikibase-user-management-area">
|
||||
<div class="wikibase-user-login" bind="loginArea">
|
||||
<div bind="invalidCredentials"></div>
|
||||
<form bind="loginForm" class="wikibase-login-form" method="post">
|
||||
<table>
|
||||
<tr>
|
||||
<td><label for="wb-username">Username:</label></td>
|
||||
<td><input name="wb-username" placeholder="Enter your username" type="text" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><label for="wb-password">Password:</label></td>
|
||||
<td><input name="wb-password" type="password" placeholder="Enter your password" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
</form>
|
||||
<div class="wikibase-login-buttons">
|
||||
<button class="button button-primary cancel-button">Close</button>
|
||||
<button class="button button-primary" bind="loginButton">Log in</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="wikibase-user-logout" bind="logoutArea">
|
||||
<p><span bind="loggedInAs">You are logged in as:</span>
|
||||
<span bind="loggedInUsername"></span></p>
|
||||
<form bind="logoutForm" method="post">
|
||||
<input type="hidden" name="wb-username" value="null" />
|
||||
<input name="wb-password" type="hidden" value="null" />
|
||||
</form>
|
||||
<div class="wikibase-login-buttons">
|
||||
<button class="button cancel-button">Cancel</button>
|
||||
<button class="button button-primary" bind="logoutButton">Log out</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -0,0 +1,75 @@
|
||||
var ManageAccountDialog = {};
|
||||
ManageAccountDialog.launch = function(logged_in_username, callback) {
|
||||
var self = this;
|
||||
var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/manage-account-dialog.html"));
|
||||
var elmts = this._elmts = DOM.bind(frame);
|
||||
|
||||
this._level = DialogSystem.showDialog(frame);
|
||||
|
||||
var dismiss = function() {
|
||||
DialogSystem.dismissUntil(self._level - 1);
|
||||
};
|
||||
|
||||
if (logged_in_username != null) {
|
||||
elmts.loginArea.hide();
|
||||
} else {
|
||||
elmts.logoutArea.hide();
|
||||
}
|
||||
|
||||
elmts.loggedInUsername.text(logged_in_username);
|
||||
|
||||
frame.find('.cancel-button').click(function() {
|
||||
dismiss();
|
||||
callback(null);
|
||||
});
|
||||
|
||||
elmts.loginButton.click(function() {
|
||||
$.post(
|
||||
"command/wikidata/login",
|
||||
elmts.loginForm.serialize(),
|
||||
function(data) {
|
||||
if (data.logged_in) {
|
||||
dismiss();
|
||||
callback(data.username);
|
||||
} else {
|
||||
elmts.invalidCredentials.text("Invalid credentials.");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
elmts.logoutButton.click(function() {
|
||||
$.post(
|
||||
"command/wikidata/login",
|
||||
"logout=true",
|
||||
function(data) {
|
||||
if (!data.logged_in) {
|
||||
dismiss();
|
||||
callback(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
ManageAccountDialog.isLoggedIn = function(callback) {
|
||||
$.get(
|
||||
"command/wikidata/login",
|
||||
function(data) {
|
||||
callback(data.username);
|
||||
});
|
||||
};
|
||||
|
||||
ManageAccountDialog.ensureLoggedIn = function(callback) {
|
||||
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
|
||||
if (logged_in_username == null) {
|
||||
ManageAccountDialog.launch(null, callback);
|
||||
} else {
|
||||
callback(logged_in_username);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
ManageAccountDialog.checkAndLaunch = function () {
|
||||
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
|
||||
ManageAccountDialog.launch(logged_in_username, function(success) { });
|
||||
});
|
||||
};
|
||||
@ -0,0 +1,21 @@
|
||||
<div class="dialog-frame" style="width: 800px;">
|
||||
<div class="dialog-header" bind="dialogHeader">Peform edits on Wikidata</div>
|
||||
<div class="dialog-body" bind="dialogBody">
|
||||
<p class="body-text">
|
||||
Please review your edits before pushing them to Wikidata.
|
||||
Consider requesting feedback at the <a href="https://www.wikidata.org/wiki/Wikidata:Data_Import_Hub">Data Import Hub</a>
|
||||
for large datasets.
|
||||
</p>
|
||||
<div class="wikibase-perform-edits-area">
|
||||
<p>You are logged in as <span bind="loggedInUsername"></span>.</p>
|
||||
<form bind="performEditsForm">
|
||||
<input type="hidden" name="strategy" value="SNAK_QUALIFIERS" />
|
||||
<input type="hidden" name="action" value="MERGE" />
|
||||
</form>
|
||||
<div class="wikibase-login-buttons">
|
||||
<button class="button cancel-button" bind="cancelButton">Cancel</button>
|
||||
<button class="button button-primary" bind="performEditsButton">Perform edits</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -0,0 +1,41 @@
|
||||
var PerformEditsDialog = {};
|
||||
|
||||
PerformEditsDialog.launch = function(logged_in_username) {
|
||||
var self = this;
|
||||
var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/perform-edits-dialog.html"));
|
||||
var elmts = this._elmts = DOM.bind(frame);
|
||||
|
||||
this._level = DialogSystem.showDialog(frame);
|
||||
|
||||
var dismiss = function() {
|
||||
DialogSystem.dismissUntil(self._level - 1);
|
||||
};
|
||||
|
||||
elmts.loggedInUsername.text(logged_in_username);
|
||||
|
||||
frame.find('.cancel-button').click(function() {
|
||||
dismiss();
|
||||
});
|
||||
|
||||
elmts.performEditsButton.click(function() {
|
||||
Refine.postProcess(
|
||||
"wikidata",
|
||||
"perform-wikibase-edits",
|
||||
{},
|
||||
elmts.performEditsForm.serialize(),
|
||||
{},
|
||||
{ onDone:
|
||||
function() {
|
||||
dismiss();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
PerformEditsDialog.checkAndLaunch = function () {
|
||||
ManageAccountDialog.ensureLoggedIn(function(logged_in_username) {
|
||||
if (logged_in_username) {
|
||||
PerformEditsDialog.launch(logged_in_username);
|
||||
}
|
||||
});
|
||||
};
|
||||
@ -1,67 +1,3 @@
|
||||
/*
|
||||
ExporterManager.MenuItems.push({});//add separator
|
||||
ExporterManager.MenuItems.push(
|
||||
{
|
||||
"id" : "exportRdfXml",
|
||||
"label":"RDF as RDF/XML",
|
||||
"click": function() { RdfExporterMenuBar.exportRDF("rdf", "rdf");}
|
||||
}
|
||||
);
|
||||
ExporterManager.MenuItems.push(
|
||||
{
|
||||
"id" : "exportRdfTurtle",
|
||||
"label":"RDF as Turtle",
|
||||
"click": function() { RdfExporterMenuBar.exportRDF("Turtle", "ttl"); }
|
||||
}
|
||||
);
|
||||
|
||||
RdfExporterMenuBar = {};
|
||||
|
||||
RdfExporterMenuBar.exportRDF = function(format, ext) {
|
||||
if (!theProject.overlayModels.rdfSchema) {
|
||||
alert(
|
||||
"You haven't done any RDF schema alignment yet!"
|
||||
);
|
||||
} else {
|
||||
RdfExporterMenuBar.rdfExportRows(format, ext);
|
||||
}
|
||||
};
|
||||
|
||||
RdfExporterMenuBar.rdfExportRows = function(format, ext) {
|
||||
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
|
||||
var form = document.createElement("form");
|
||||
$(form)
|
||||
.css("display", "none")
|
||||
.attr("method", "post")
|
||||
.attr("action", "command/core/export-rows/" + name + "." + ext)
|
||||
.attr("target", "gridworks-export");
|
||||
|
||||
$('<input />')
|
||||
.attr("name", "engine")
|
||||
.attr("value", JSON.stringify(ui.browsingEngine.getJSON()))
|
||||
.appendTo(form);
|
||||
$('<input />')
|
||||
.attr("name", "project")
|
||||
.attr("value", theProject.id)
|
||||
.appendTo(form);
|
||||
$('<input />')
|
||||
.attr("name", "format")
|
||||
.attr("value", format)
|
||||
.appendTo(form);
|
||||
|
||||
document.body.appendChild(form);
|
||||
|
||||
window.open("about:blank", "gridworks-export");
|
||||
form.submit();
|
||||
|
||||
document.body.removeChild(form);
|
||||
};
|
||||
|
||||
RdfExporterMenuBar.editRdfSchema = function(reset) {
|
||||
new RdfSchemaAlignmentDialog(reset ? null : theProject.overlayModels.rdfSchema);
|
||||
};
|
||||
*/
|
||||
|
||||
ExporterManager.MenuItems.push({});
|
||||
ExporterManager.MenuItems.push(
|
||||
{
|
||||
@ -109,15 +45,26 @@ $(function(){
|
||||
"label": "Wikidata",
|
||||
"submenu" : [
|
||||
{
|
||||
"id": "wikidata/edit-schema",
|
||||
id: "wikidata/edit-schema",
|
||||
label: "Edit Wikibase schema...",
|
||||
click: function() { SchemaAlignmentDialog.launch(false); }
|
||||
},
|
||||
{
|
||||
id:"wikidata/manage-account",
|
||||
label: "Manage account",
|
||||
click: function() { ManageAccountDialog.checkAndLaunch(); }
|
||||
},
|
||||
{
|
||||
id:"wikidata/perform-edits",
|
||||
label: "Push to Wikidata...",
|
||||
click: function() { PerformEditsDialog.checkAndLaunch(); }
|
||||
},
|
||||
{
|
||||
"id":"wikidata/export-qs",
|
||||
"label": "Export to QuickStatements",
|
||||
"click": function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
|
||||
}
|
||||
id:"wikidata/export-qs",
|
||||
label: "Export to QuickStatements",
|
||||
click: function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
|
||||
},
|
||||
|
||||
]
|
||||
}
|
||||
);
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
@import-less url("../theme.less");
|
||||
|
||||
.wikibase-login-form {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.wikibase-login-buttons {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
@ -0,0 +1,53 @@
|
||||
package org.openrefine.wikidata.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
import org.openrefine.wikidata.editing.ConnectionManager;
|
||||
|
||||
import com.google.refine.commands.Command;
|
||||
|
||||
|
||||
public class LoginCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
String username = request.getParameter("wb-username");
|
||||
String password = request.getParameter("wb-password");
|
||||
ConnectionManager manager = ConnectionManager.getInstance();
|
||||
if (username != null && password != null) {
|
||||
manager.login(username, password);
|
||||
} else if ("true".equals(request.getParameter("logout"))) {
|
||||
manager.logout();
|
||||
}
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
StringWriter sb = new StringWriter(2048);
|
||||
JSONWriter writer = new JSONWriter(sb, 32);
|
||||
|
||||
try {
|
||||
writer.object();
|
||||
writer.key("logged_in");
|
||||
writer.value(manager.isLoggedIn());
|
||||
writer.key("username");
|
||||
writer.value(manager.getUsername());
|
||||
writer.endObject();
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
respond(response, sb.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package org.openrefine.wikidata.commands;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation;
|
||||
import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation.DuplicateDetectionStrategy;
|
||||
import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation.OnDuplicateAction;
|
||||
|
||||
import com.google.refine.commands.EngineDependentCommand;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.Project;
|
||||
|
||||
public class PerformWikibaseEditsCommand extends EngineDependentCommand {
|
||||
|
||||
@Override
|
||||
protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig)
|
||||
throws Exception {
|
||||
String strategy = request.getParameter("strategy");
|
||||
String action = request.getParameter("action");
|
||||
return new PerformWikibaseEditsOperation(engineConfig,
|
||||
DuplicateDetectionStrategy.valueOf(strategy),
|
||||
OnDuplicateAction.valueOf(action));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
package org.openrefine.wikidata.editing;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
||||
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.preference.PreferenceStore;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Manages a connection to Wikidata, with login credentials stored
|
||||
* in the preferences.
|
||||
*
|
||||
* Ideally, we should store only the cookies and not the password.
|
||||
* But Wikidata-Toolkit does not allow for that as cookies are kept
|
||||
* private.
|
||||
*
|
||||
* This class is also hard-coded for Wikidata: generalization to other
|
||||
* Wikibase instances should be feasible though.
|
||||
*
|
||||
* @author antonin
|
||||
*/
|
||||
|
||||
public class ConnectionManager {
|
||||
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
|
||||
|
||||
private PreferenceStore prefStore;
|
||||
private ApiConnection connection;
|
||||
|
||||
private static class ConnectionManagerHolder {
|
||||
private static final ConnectionManager instance = new ConnectionManager();
|
||||
}
|
||||
|
||||
public static ConnectionManager getInstance() {
|
||||
return ConnectionManagerHolder.instance;
|
||||
}
|
||||
|
||||
private ConnectionManager() {
|
||||
prefStore = ProjectManager.singleton.getPreferenceStore();
|
||||
connection = null;
|
||||
}
|
||||
|
||||
public void login(String username, String password) {
|
||||
try {
|
||||
JSONArray array = new JSONArray();
|
||||
JSONObject obj = new JSONObject();
|
||||
obj.put("username", username);
|
||||
obj.put("password", password);
|
||||
array.put(obj);
|
||||
prefStore.put(PREFERENCE_STORE_KEY, array);
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
connection = ApiConnection.getWikidataApiConnection();
|
||||
try {
|
||||
connection.login(username, password);
|
||||
} catch (LoginFailedException e) {
|
||||
connection = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void restorePreviousLogin() {
|
||||
JSONArray array = (JSONArray) prefStore.get(PREFERENCE_STORE_KEY);
|
||||
if (array.length() > 0) {
|
||||
JSONObject obj;
|
||||
try {
|
||||
obj = array.getJSONObject(0);
|
||||
String username = obj.getString("username");
|
||||
String password = obj.getString("password");
|
||||
login(username, password);
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void logout() {
|
||||
prefStore.put(PREFERENCE_STORE_KEY, new JSONArray());
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.logout();
|
||||
connection = null;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ApiConnection getConnection() {
|
||||
return connection;
|
||||
}
|
||||
|
||||
public boolean isLoggedIn() {
|
||||
return connection != null;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
if (connection != null) {
|
||||
return connection.getCurrentUser();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
package org.openrefine.wikidata.editing;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
|
||||
/**
|
||||
* This is just the necessary bits to store Wikidata credentials
|
||||
* in OpenRefine's preference store.
|
||||
*
|
||||
* @author antonin
|
||||
*
|
||||
*/
|
||||
class WikibaseCredentials implements Jsonizable {
|
||||
|
||||
private String username;
|
||||
private String password;
|
||||
|
||||
public WikibaseCredentials() {
|
||||
username = null;
|
||||
password = null;
|
||||
}
|
||||
|
||||
public WikibaseCredentials(String username, String password) {
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public boolean isNonNull() {
|
||||
return username != null && password != null && ! "null".equals(username) && ! "null".equals(password);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
writer.object();
|
||||
writer.key("class");
|
||||
writer.value(this.getClass().getName());
|
||||
writer.key("username");
|
||||
writer.value(username);
|
||||
writer.key("password");
|
||||
writer.value(password);
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public static WikibaseCredentials load(JSONObject obj) throws JSONException {
|
||||
return new WikibaseCredentials(
|
||||
obj.getString("username"),
|
||||
obj.getString("password"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.exporters.WriterExporter;
|
||||
import com.google.refine.model.Project;
|
||||
|
||||
import org.openrefine.wikidata.schema.ItemUpdate;
|
||||
import org.openrefine.wikidata.schema.WikibaseSchema;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@ -16,7 +17,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
|
||||
@ -24,7 +24,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Reference;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.Snak;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.Value;
|
||||
@ -54,29 +53,25 @@ public class QuickStatementsExporter implements WriterExporter {
|
||||
}
|
||||
|
||||
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer) throws IOException {
|
||||
List<ItemDocument> items = schema.evaluate(project, engine);
|
||||
for (ItemDocument item : items) {
|
||||
List<ItemUpdate> items = schema.evaluate(project, engine);
|
||||
for (ItemUpdate item : items) {
|
||||
translateItem(item, writer);
|
||||
}
|
||||
}
|
||||
|
||||
protected void translateItem(ItemDocument item, Writer writer) throws IOException {
|
||||
protected void translateItem(ItemUpdate item, Writer writer) throws IOException {
|
||||
if (item.getItemId().equals(ItemIdValue.NULL)) {
|
||||
writer.write("CREATE\n");
|
||||
}
|
||||
for (StatementGroup group : item.getStatementGroups()) {
|
||||
translateStatementGroup(group, writer);
|
||||
for (Statement s : item.getAddedStatements()) {
|
||||
translateStatement(s, s.getClaim().getMainSnak().getPropertyId().getId(), true, writer);
|
||||
}
|
||||
for (Statement s : item.getDeletedStatements()) {
|
||||
translateStatement(s, s.getClaim().getMainSnak().getPropertyId().getId(), false, writer);
|
||||
}
|
||||
}
|
||||
|
||||
protected void translateStatementGroup(StatementGroup group, Writer writer) throws IOException {
|
||||
String pid = group.getProperty().getId();
|
||||
for(Statement statement : group.getStatements()) {
|
||||
translateStatement(statement, pid, writer);
|
||||
}
|
||||
}
|
||||
|
||||
protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException {
|
||||
protected void translateStatement(Statement statement, String pid, boolean add, Writer writer) throws IOException {
|
||||
Claim claim = statement.getClaim();
|
||||
String qid = claim.getSubject().getId();
|
||||
if (claim.getSubject().equals(ItemIdValue.NULL)) {
|
||||
@ -86,6 +81,9 @@ public class QuickStatementsExporter implements WriterExporter {
|
||||
ValueVisitor<String> vv = new ValuePrinter();
|
||||
String targetValue = val.accept(vv);
|
||||
if (targetValue != null) {
|
||||
if (! add) {
|
||||
writer.write("- ");
|
||||
}
|
||||
writer.write(qid + "\t" + pid + "\t" + targetValue);
|
||||
for(SnakGroup q : claim.getQualifiers()) {
|
||||
translateSnakGroup(q, false, writer);
|
||||
|
||||
@ -0,0 +1,242 @@
|
||||
package org.openrefine.wikidata.operations;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.LineNumberReader;
|
||||
import java.io.Writer;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.openrefine.wikidata.editing.ConnectionManager;
|
||||
import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation.WikibaseSchemaChange;
|
||||
import org.openrefine.wikidata.schema.ItemUpdate;
|
||||
import org.openrefine.wikidata.schema.WikibaseSchema;
|
||||
import org.wikidata.wdtk.datamodel.helpers.EntityDocumentBuilder;
|
||||
import org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||
import org.wikidata.wdtk.util.WebResourceFetcherImpl;
|
||||
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
||||
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
|
||||
import org.wikidata.wdtk.wikibaseapi.StatementUpdate;
|
||||
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
|
||||
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
|
||||
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.history.Change;
|
||||
import com.google.refine.history.HistoryEntry;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.changes.ReconChange;
|
||||
import com.google.refine.operations.EngineDependentOperation;
|
||||
import com.google.refine.operations.OperationRegistry;
|
||||
import com.google.refine.operations.recon.ReconOperation;
|
||||
import com.google.refine.process.LongRunningProcess;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.util.ParsingUtilities;
|
||||
import com.google.refine.util.Pool;
|
||||
|
||||
|
||||
public class PerformWikibaseEditsOperation extends EngineDependentOperation {
|
||||
public enum DuplicateDetectionStrategy {
|
||||
PROPERTY, SNAK, SNAK_QUALIFIERS
|
||||
}
|
||||
|
||||
public enum OnDuplicateAction {
|
||||
SKIP, MERGE
|
||||
}
|
||||
|
||||
private DuplicateDetectionStrategy strategy;
|
||||
private OnDuplicateAction duplicateAction;
|
||||
|
||||
public PerformWikibaseEditsOperation(
|
||||
JSONObject engineConfig,
|
||||
DuplicateDetectionStrategy strategy,
|
||||
OnDuplicateAction duplicateAction) {
|
||||
super(engineConfig);
|
||||
this.strategy = strategy;
|
||||
this.duplicateAction = duplicateAction;
|
||||
|
||||
// getEngine(request, project);
|
||||
}
|
||||
|
||||
static public AbstractOperation reconstruct(Project project, JSONObject obj)
|
||||
throws Exception {
|
||||
JSONObject engineConfig = obj.getJSONObject("engineConfig");
|
||||
String strategy = obj.getString("duplicate_strategy");
|
||||
String action = obj.getString("duplicate_action");
|
||||
return new PerformWikibaseEditsOperation(
|
||||
engineConfig,
|
||||
DuplicateDetectionStrategy.valueOf(strategy),
|
||||
OnDuplicateAction.valueOf(action));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
writer.object();
|
||||
writer.key("op");
|
||||
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
|
||||
writer.key("description");
|
||||
writer.value("Perform Wikibase edits");
|
||||
writer.key("duplicate_strategy");
|
||||
writer.value(strategy.name());
|
||||
writer.key("duplicate_action");
|
||||
writer.value(duplicateAction.name());
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBriefDescription(Project project) {
|
||||
return "Peform edits on Wikidata";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Process createProcess(Project project, Properties options) throws Exception {
|
||||
return new PerformEditsProcess(
|
||||
project,
|
||||
createEngine(project),
|
||||
getBriefDescription(project),
|
||||
"#openrefine"
|
||||
);
|
||||
}
|
||||
static public class PerformWikibaseEditsChange implements Change {
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
// this does not do anything to the project (we don't re-run changes on Wikidata)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void revert(Project project) {
|
||||
// this does not do anything (we don't revert changes on Wikidata either)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save(Writer writer, Properties options)
|
||||
throws IOException {
|
||||
writer.write("/ec/\n"); // end of change
|
||||
}
|
||||
|
||||
static public Change load(LineNumberReader reader, Pool pool)
|
||||
throws Exception {
|
||||
return new PerformWikibaseEditsChange();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class PerformEditsProcess extends LongRunningProcess implements Runnable {
|
||||
|
||||
protected Project _project;
|
||||
protected Engine _engine;
|
||||
protected WikibaseSchema _schema;
|
||||
protected String _summary;
|
||||
protected final long _historyEntryID;
|
||||
|
||||
protected PerformEditsProcess(Project project,
|
||||
Engine engine, String description, String summary) {
|
||||
super(description);
|
||||
this._project = project;
|
||||
this._engine = engine;
|
||||
this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
|
||||
this._summary = summary;
|
||||
this._historyEntryID = HistoryEntry.allocateID();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension");
|
||||
ConnectionManager manager = ConnectionManager.getInstance();
|
||||
if (!manager.isLoggedIn()) {
|
||||
return;
|
||||
}
|
||||
ApiConnection connection = manager.getConnection();
|
||||
|
||||
//WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, schema.getBaseUri());
|
||||
WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseUri());
|
||||
//wbde.disableEditing();
|
||||
|
||||
// Evaluate the schema
|
||||
List<ItemUpdate> itemDocuments = _schema.evaluate(_project, _engine);
|
||||
|
||||
// Group statements by item
|
||||
Map<EntityIdValue, ItemUpdate> updates = new HashMap<EntityIdValue, ItemUpdate>();
|
||||
for(ItemUpdate update : itemDocuments) {
|
||||
if (update.isNull()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ItemIdValue qid = update.getItemId();
|
||||
if (updates.containsKey(qid)) {
|
||||
ItemUpdate oldUpdate = updates.get(qid);
|
||||
oldUpdate.merge(update);
|
||||
} else {
|
||||
updates.put(qid, update);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO:
|
||||
* - support for new items
|
||||
* - support for duplicate strategy and action
|
||||
*/
|
||||
|
||||
// Perform edits
|
||||
int totalItemUpdates = updates.size();
|
||||
int updatesDone = 0;
|
||||
for(ItemUpdate update : updates.values()) {
|
||||
try {
|
||||
wbde.updateStatements(update.getItemId(), update.getAddedStatements(), update.getDeletedStatements(), _summary);
|
||||
|
||||
} catch (MediaWikiApiErrorException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
updatesDone++;
|
||||
_progress = (100*updatesDone) / totalItemUpdates;
|
||||
|
||||
if(_canceled) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
_progress = 100;
|
||||
|
||||
if (!_canceled) {
|
||||
Change change = new PerformWikibaseEditsChange();
|
||||
|
||||
HistoryEntry historyEntry = new HistoryEntry(
|
||||
_historyEntryID,
|
||||
_project,
|
||||
_description,
|
||||
PerformWikibaseEditsOperation.this,
|
||||
change
|
||||
);
|
||||
|
||||
_project.history.addEntry(historyEntry);
|
||||
_project.processManager.onDoneProcess(this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Runnable getRunnable() {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,68 @@
|
||||
package org.openrefine.wikidata.schema;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||
|
||||
|
||||
/**
|
||||
* A class to plan an update of an item, after evaluating the statements
|
||||
* but before fetching the current content of the item (this is why it does not
|
||||
* extend StatementsUpdate).
|
||||
*
|
||||
* @author antonin
|
||||
*/
|
||||
public class ItemUpdate {
|
||||
private ItemIdValue qid;
|
||||
private List<Statement> addedStatements;
|
||||
private List<Statement> deletedStatements;
|
||||
|
||||
public ItemUpdate(ItemIdValue qid) {
|
||||
this.qid = qid;
|
||||
this.addedStatements = new ArrayList<Statement>();
|
||||
this.deletedStatements = new ArrayList<Statement>();
|
||||
}
|
||||
|
||||
public void addStatement(Statement s) {
|
||||
addedStatements.add(s);
|
||||
}
|
||||
|
||||
public void deleteStatement(Statement s) {
|
||||
deletedStatements.add(s);
|
||||
}
|
||||
|
||||
public void addStatements(List<Statement> l) {
|
||||
addedStatements.addAll(l);
|
||||
}
|
||||
|
||||
public void deleteStatements(List<Statement> l) {
|
||||
deletedStatements.addAll(l);
|
||||
}
|
||||
|
||||
public ItemIdValue getItemId() {
|
||||
return qid;
|
||||
}
|
||||
|
||||
public List<Statement> getAddedStatements() {
|
||||
return addedStatements;
|
||||
}
|
||||
|
||||
public List<Statement> getDeletedStatements() {
|
||||
return deletedStatements;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges all the changes in other into this instance.
|
||||
* @param other: the other change that should be merged
|
||||
*/
|
||||
public void merge(ItemUpdate other) {
|
||||
addStatements(other.getAddedStatements());
|
||||
deleteStatements(other.getDeletedStatements());
|
||||
}
|
||||
|
||||
public boolean isNull() {
|
||||
return addedStatements.isEmpty() && deletedStatements.isEmpty();
|
||||
}
|
||||
}
|
||||
@ -52,15 +52,15 @@ public class WbItemDocumentExpr extends BiJsonizable {
|
||||
statementExprs);
|
||||
}
|
||||
|
||||
public ItemDocument evaluate(ExpressionContext ctxt) throws SkipStatementException {
|
||||
public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipStatementException {
|
||||
ItemIdValue subjectId = subjectExpr.evaluate(ctxt);
|
||||
ItemDocumentBuilder builder = ItemDocumentBuilder.forItemId(subjectId);
|
||||
ItemUpdate update = new ItemUpdate(subjectId);
|
||||
for(WbStatementGroupExpr expr : statementGroupExprs) {
|
||||
for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
|
||||
builder.withStatement(s);
|
||||
update.addStatement(s);
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
return update;
|
||||
}
|
||||
|
||||
public String getJsonType() {
|
||||
|
||||
@ -58,8 +58,8 @@ public class WikibaseSchema implements OverlayModel {
|
||||
return itemDocumentExprs;
|
||||
}
|
||||
|
||||
public List<ItemDocument> evaluate(ExpressionContext ctxt) {
|
||||
List<ItemDocument> result = new ArrayList<ItemDocument>();
|
||||
public List<ItemUpdate> evaluate(ExpressionContext ctxt) {
|
||||
List<ItemUpdate> result = new ArrayList<ItemUpdate>();
|
||||
for (WbItemDocumentExpr expr : itemDocumentExprs) {
|
||||
|
||||
try {
|
||||
@ -71,16 +71,16 @@ public class WikibaseSchema implements OverlayModel {
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<ItemDocument> evaluate(Project project, Engine engine) {
|
||||
List<ItemDocument> result = new ArrayList<ItemDocument>();
|
||||
public List<ItemUpdate> evaluate(Project project, Engine engine) {
|
||||
List<ItemUpdate> result = new ArrayList<ItemUpdate>();
|
||||
FilteredRows filteredRows = engine.getAllFilteredRows();
|
||||
filteredRows.accept(project, new EvaluatingRowVisitor(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
protected class EvaluatingRowVisitor implements RowVisitor {
|
||||
private List<ItemDocument> result;
|
||||
public EvaluatingRowVisitor(List<ItemDocument> result) {
|
||||
private List<ItemUpdate> result;
|
||||
public EvaluatingRowVisitor(List<ItemUpdate> result) {
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user