s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add license headers, general cleanups on Java files

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-03-04 00:14:59 +00:00
parent d6b229f25e
commit 0b14a1a627
140 changed files with 5494 additions and 2592 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
extensions/wikidata/credits.txt
View File

@ -1,10 +1,32 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2018 Antonin Delpeuch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#-------------------------------------------------------------------------------
Sources for the images:
- Information.svg by User:Bobarino:
The following images were obtained from Wikimedia Commons:
- Information.svg by User:Bobarino: (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information.svg
- Warning.svg by User:Ezekiel63745
- Warning.svg by User:Ezekiel63745 (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information_orange.svg
- Important.svg originally from David Vignoni
- Important.svg originally from David Vignoni (GNU LGPL)
https://commons.wikimedia.org/wiki/File:Nuvola_apps_important.svg
- Critical.svg by User:Stannered
- Critical.svg by User:Stannered (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Stop_x_nuvola.svg

31
extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
@ -14,13 +37,13 @@ import org.openrefine.wikidata.editing.ConnectionManager;
import com.google.refine.commands.Command;
public class LoginCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String username = request.getParameter("wb-username");
String password = request.getParameter("wb-password");
String remember = request.getParameter("remember-credentials");
System.out.println(remember);
ConnectionManager manager = ConnectionManager.getInstance();
if (username != null && password != null) {
manager.login(username, password, "on".equals(remember));
@ -29,10 +52,10 @@ public class LoginCommand extends Command {
}
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
try {
writer.object();
writer.key("logged_in");
@ -45,7 +68,7 @@ public class LoginCommand extends Command {
}
respond(response, sb.toString());
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {

26
extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import javax.servlet.http.HttpServletRequest;
@ -15,8 +38,7 @@ public class PerformWikibaseEditsCommand extends EngineDependentCommand {
protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig)
throws Exception {
String summary = request.getParameter("summary");
return new PerformWikibaseEditsOperation(engineConfig,
summary);
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
}

57
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
/*
Copyright 2010, Google Inc.
@ -34,8 +57,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
@ -63,25 +84,25 @@ import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class PreviewWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("schema");
WikibaseSchema schema = null;
if (jsonString != null) {
try {
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
schema = WikibaseSchema.reconstruct(json);
} catch(JSONException e) {
} catch (JSONException e) {
respond(response, "error", "Wikibase schema could not be parsed.");
return;
}
@ -92,20 +113,20 @@ public class PreviewWikibaseSchemaCommand extends Command {
respond(response, "error", "No Wikibase schema provided.");
return;
}
QAWarningStore warningStore = new QAWarningStore();
// Evaluate project
Engine engine = getEngine(request, project);
List<ItemUpdate> editBatch = schema.evaluate(project, engine, warningStore);
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
writer.object();
{
StringWriter stringWriter = new StringWriter();
// Inspect the edits and generate warnings
EditInspector inspector = new EditInspector(warningStore);
inspector.inspect(editBatch);
@ -115,22 +136,22 @@ public class PreviewWikibaseSchemaCommand extends Command {
warning.write(writer, new Properties());
}
writer.endArray();
// this is not the length of the warnings array written before,
// but the total number of issues raised (before deduplication)
writer.key("nb_warnings");
writer.value(warningStore.getNbWarnings());
// Export to QuickStatements
QuickStatementsExporter exporter = new QuickStatementsExporter();
QuickStatementsExporter exporter = new QuickStatementsExporter();
exporter.translateItemList(editBatch, stringWriter);
writer.key("quickstatements");
writer.value(FirstLinesExtractor.extractFirstLines(stringWriter.toString(), 50));
}
writer.endObject();
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);

35
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
@ -23,24 +46,24 @@ public class SaveWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("schema");
if (jsonString == null) {
respond(response, "error", "No Wikibase schema provided.");
return;
}
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
WikibaseSchema schema = WikibaseSchema.reconstruct(json);
AbstractOperation op = new SaveWikibaseSchemaOperation(schema);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (JSONException e) {
respond(response, "error", "Wikibase schema could not be parsed.");
} catch (Exception e) {

71
extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java
View File

@ -1,54 +1,74 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
/**
* Manages a connection to Wikidata, with login credentials stored
* in the preferences.
* Manages a connection to Wikidata, with login credentials stored in the
* preferences.
*
* Ideally, we should store only the cookies and not the password.
* But Wikidata-Toolkit does not allow for that as cookies are kept
* private.
* Ideally, we should store only the cookies and not the password. But
* Wikidata-Toolkit does not allow for that as cookies are kept private.
*
* This class is also hard-coded for Wikidata: generalization to other
* Wikibase instances should be feasible though.
* This class is also hard-coded for Wikidata: generalization to other Wikibase
* instances should be feasible though.
*
* @author antonin
* @author Antonin Delpeuch
*/
public class ConnectionManager {
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
private PreferenceStore prefStore;
private ApiConnection connection;
private static class ConnectionManagerHolder {
private static final ConnectionManager instance = new ConnectionManager();
}
public static ConnectionManager getInstance() {
return ConnectionManagerHolder.instance;
}
private ConnectionManager() {
prefStore = ProjectManager.singleton.getPreferenceStore();
connection = null;
restoreSavedConnection();
}
public void login(String username, String password, boolean rememberCredentials) {
if (rememberCredentials) {
try {
@ -63,7 +83,7 @@ public class ConnectionManager {
e.printStackTrace();
}
}
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(username, password);
@ -71,14 +91,13 @@ public class ConnectionManager {
connection = null;
}
}
public void restoreSavedConnection() {
JSONObject savedCredentials = getStoredCredentials();
if (savedCredentials != null) {
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(savedCredentials.getString("username"),
savedCredentials.getString("password"));
connection.login(savedCredentials.getString("username"), savedCredentials.getString("password"));
} catch (LoginFailedException e) {
connection = null;
} catch (JSONException e) {
@ -86,7 +105,7 @@ public class ConnectionManager {
}
}
}
public JSONObject getStoredCredentials() {
JSONArray array = (JSONArray) prefStore.get(PREFERENCE_STORE_KEY);
if (array.length() > 0) {
@ -94,11 +113,11 @@ public class ConnectionManager {
return array.getJSONObject(0);
} catch (JSONException e) {
e.printStackTrace();
}
}
}
return null;
}
public void logout() {
prefStore.put(PREFERENCE_STORE_KEY, new JSONArray());
if (connection != null) {
@ -110,11 +129,11 @@ public class ConnectionManager {
}
}
}
public ApiConnection getConnection() {
return connection;
}
public boolean isLoggedIn() {
return connection != null;
}

352
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
@ -27,184 +50,161 @@ import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
*
*/
public class EditBatchProcessor {
static final Logger logger = LoggerFactory
.getLogger(EditBatchProcessor.class);
private WikibaseDataFetcher fetcher;
private WikibaseDataEditor editor;
private NewItemLibrary library;
private List<ItemUpdate> scheduled;
private String summary;
private List<ItemUpdate> remainingUpdates;
private List<ItemUpdate> currentBatch;
private int batchCursor;
private int globalCursor;
private Map<String, EntityDocument> currentDocs;
private int batchSize;
/**
* Initiates the process of pushing a batch of updates
* to Wikibase. This schedules the updates and is a
* prerequisite for calling {@link performOneEdit}.
*
* @param fetcher
* the fetcher to use to retrieve the current state of items
* @param editor
* the object to use to perform the edits
* @param updates
* the list of item updates to perform
* @param library
* the library to use to keep track of new item creation
* @param summary
* the summary to append to all edits
* @param batchSize
* the number of items that should be retrieved in one go from the API
*/
public EditBatchProcessor(WikibaseDataFetcher fetcher,
WikibaseDataEditor editor,
List<ItemUpdate> updates,
NewItemLibrary library,
String summary,
int batchSize) {
this.fetcher = fetcher;
this.editor = editor;
editor.setEditAsBot(true); // this will not do anything if the user does not
// have a bot flag, and this is generally wanted if they have one.
this.library = library;
this.summary = summary;
this.batchSize = batchSize;
// Schedule the edit batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
this.scheduled = scheduler.schedule(updates);
this.globalCursor = 0;
this.batchCursor = 0;
this.remainingUpdates = new ArrayList<>(scheduled);
this.currentBatch = Collections.emptyList();
this.currentDocs = Collections.emptyMap();
}
static final Logger logger = LoggerFactory.getLogger(EditBatchProcessor.class);
/**
* Performs the next edit in the batch.
*
* @throws InterruptedException
*/
public void performEdit() throws InterruptedException {
if (remainingEdits() == 0) {
return;
}
if (batchCursor == currentBatch.size()) {
prepareNewBatch();
}
ItemUpdate update = currentBatch.get(batchCursor);
// Rewrite mentions to new items
ReconEntityRewriter rewriter = new ReconEntityRewriter(library, update.getItemId());
update = rewriter.rewrite(update);
try {
// New item
if (update.isNew()) {
ReconEntityIdValue newCell = (ReconEntityIdValue)update.getItemId();
update = update.normalizeLabelsAndAliases();
ItemDocument itemDocument = Datamodel.makeItemDocument(update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
library.setQid(newCell.getReconInternalId(), createdDoc.getItemId().getId());
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument)currentDocs.get(update.getItemId().getId());
/*
TermStatementUpdate tsUpdate = new TermStatementUpdate(
currentDocument,
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>()
);
ObjectMapper mapper = new ObjectMapper();
logger.info(mapper.writeValueAsString(update));
logger.info(update.toString());
logger.info(tsUpdate.getJsonUpdateString()); */
editor.updateTermsStatements(currentDocument,
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()),
summary);
}
} catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
batchCursor++;
}
/**
* @return the number of edits that remain to be done in the current batch
*/
public int remainingEdits() {
return scheduled.size() - (globalCursor + batchCursor);
}
/**
* @return the progress, measured as a percentage
*/
public int progress() {
return (100*(globalCursor + batchCursor)) / scheduled.size();
}
protected void prepareNewBatch() throws InterruptedException {
// remove the previous batch from the remainingUpdates
globalCursor += currentBatch.size();
currentBatch.clear();
if(remainingUpdates.size() < batchSize) {
private WikibaseDataFetcher fetcher;
private WikibaseDataEditor editor;
private NewItemLibrary library;
private List<ItemUpdate> scheduled;
private String summary;
private List<ItemUpdate> remainingUpdates;
private List<ItemUpdate> currentBatch;
private int batchCursor;
private int globalCursor;
private Map<String, EntityDocument> currentDocs;
private int batchSize;
/**
* Initiates the process of pushing a batch of updates to Wikibase. This
* schedules the updates and is a prerequisite for calling
* {@link performOneEdit}.
*
* @param fetcher
* the fetcher to use to retrieve the current state of items
* @param editor
* the object to use to perform the edits
* @param updates
* the list of item updates to perform
* @param library
* the library to use to keep track of new item creation
* @param summary
* the summary to append to all edits
* @param batchSize
* the number of items that should be retrieved in one go from the
* API
*/
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
NewItemLibrary library, String summary, int batchSize) {
this.fetcher = fetcher;
this.editor = editor;
editor.setEditAsBot(true); // this will not do anything if the user does not
// have a bot flag, and this is generally wanted if they have one.
this.library = library;
this.summary = summary;
this.batchSize = batchSize;
// Schedule the edit batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
this.scheduled = scheduler.schedule(updates);
this.globalCursor = 0;
this.batchCursor = 0;
this.remainingUpdates = new ArrayList<>(scheduled);
this.currentBatch = Collections.emptyList();
this.currentDocs = Collections.emptyMap();
}
/**
* Performs the next edit in the batch.
*
* @throws InterruptedException
*/
public void performEdit()
throws InterruptedException {
if (remainingEdits() == 0) {
return;
}
if (batchCursor == currentBatch.size()) {
prepareNewBatch();
}
ItemUpdate update = currentBatch.get(batchCursor);
// Rewrite mentions to new items
ReconEntityRewriter rewriter = new ReconEntityRewriter(library, update.getItemId());
update = rewriter.rewrite(update);
try {
// New item
if (update.isNew()) {
ReconEntityIdValue newCell = (ReconEntityIdValue) update.getItemId();
update = update.normalizeLabelsAndAliases();
ItemDocument itemDocument = Datamodel.makeItemDocument(update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
library.setQid(newCell.getReconInternalId(), createdDoc.getItemId().getId());
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
editor.updateTermsStatements(currentDocument, update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()), summary);
}
} catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
batchCursor++;
}
/**
* @return the number of edits that remain to be done in the current batch
*/
public int remainingEdits() {
return scheduled.size() - (globalCursor + batchCursor);
}
/**
* @return the progress, measured as a percentage
*/
public int progress() {
return (100 * (globalCursor + batchCursor)) / scheduled.size();
}
protected void prepareNewBatch()
throws InterruptedException {
// remove the previous batch from the remainingUpdates
globalCursor += currentBatch.size();
currentBatch.clear();
if (remainingUpdates.size() < batchSize) {
currentBatch = remainingUpdates;
remainingUpdates = Collections.emptyList();
} else {
currentBatch = remainingUpdates.subList(0, batchSize);
}
List<String> qidsToFetch = currentBatch.stream()
.filter(u -> !u.isNew())
.map(u -> u.getItemId().getId())
.collect(Collectors.toList());
// Get the current documents for this batch of updates
logger.info("Requesting documents");
currentDocs = null;
int retries = 3;
while (currentDocs == null && retries > 0) {
try {
currentDocs = fetcher.getEntityDocuments(qidsToFetch);
} catch (MediaWikiApiErrorException e) {
e.printStackTrace();
Thread.sleep(5000);
}
retries--;
}
if (currentDocs == null) {
throw new InterruptedException("Fetching current documents failed.");
}
batchCursor = 0;
}
} else {
currentBatch = remainingUpdates.subList(0, batchSize);
}
List<String> qidsToFetch = currentBatch.stream().filter(u -> !u.isNew()).map(u -> u.getItemId().getId())
.collect(Collectors.toList());
// Get the current documents for this batch of updates
logger.info("Requesting documents");
currentDocs = null;
int retries = 3;
while (currentDocs == null && retries > 0) {
try {
currentDocs = fetcher.getEntityDocuments(qidsToFetch);
} catch (MediaWikiApiErrorException e) {
e.printStackTrace();
Thread.sleep(5000);
}
retries--;
}
if (currentDocs == null) {
throw new InterruptedException("Fetching current documents failed.");
}
batchCursor = 0;
}
}

119
extensions/wikidata/src/org/openrefine/wikidata/editing/NewItemLibrary.java
View File

@ -1,98 +1,120 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import com.google.refine.model.Project;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
/**
* This keeps track of the new items that we
* have created for each internal reconciliation id.
* This keeps track of the new items that we have created for each internal
* reconciliation id.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class NewItemLibrary {
private Map<Long, String> map;
public NewItemLibrary() {
map = new HashMap<>();
}
@JsonCreator
public NewItemLibrary(@JsonProperty("qidMap") Map<Long, String> map) {
this.map = map;
}
/**
* Retrieves the Qid allocated to a given new cell
* @param id: the fake ItemId generated by the cell
*
* @param id:
* the fake ItemId generated by the cell
* @return the qid (or null if unallocated yet)
*/
public String getQid(long id) {
return map.get(id);
}
/**
* Stores a Qid associated to a new cell
* @param id : the internal reconciliation id of the new cell
* @param qid : the associated Qid returned by Wikibase
*
* @param id
* : the internal reconciliation id of the new cell
* @param qid
* : the associated Qid returned by Wikibase
*/
public void setQid(long id, String qid) {
map.put(id, qid);
}
/**
* Changes the "new" reconciled cells to their allocated
* qids for later use.
* Changes the "new" reconciled cells to their allocated qids for later use.
*
* @param reset: set to true to revert the operation (set cells to "new")
* @param reset:
* set to true to revert the operation (set cells to "new")
*/
public void updateReconciledCells(Project project, boolean reset) {
Set<Integer> impactedColumns = new HashSet<>();
Set<Integer> impactedColumns = new HashSet<>();
/*
* Note that there is a slight violation of OpenRefine's model here:
* if we reconcile multiple cells to the same new item, and then
* perform this operation on a subset of the corresponding rows,
* we are going to modify cells that are outside the facet (because
* they are reconciled to the same cell). But I think this is the
* right thing to do.
* Note that there is a slight violation of OpenRefine's model here: if we
* reconcile multiple cells to the same new item, and then perform this
* operation on a subset of the corresponding rows, we are going to modify cells
* that are outside the facet (because they are reconciled to the same cell).
* But I think this is the right thing to do.
*/
for(Row row : project.rows) {
for(int i = 0; i != row.cells.size(); i++) {
for (Row row : project.rows) {
for (int i = 0; i != row.cells.size(); i++) {
Cell cell = row.cells.get(i);
if (cell == null || cell.recon == null) {
continue;
}
Recon recon = cell.recon;
if (Recon.Judgment.New.equals(recon.judgment) && !reset &&
map.containsKey(recon.judgmentHistoryEntry)) {
if (Recon.Judgment.New.equals(recon.judgment) && !reset
&& map.containsKey(recon.judgmentHistoryEntry)) {
recon.judgment = Recon.Judgment.Matched;
recon.match = new ReconCandidate(
map.get(recon.judgmentHistoryEntry),
cell.value.toString(),
new String[0],
100);
recon.match = new ReconCandidate(map.get(recon.judgmentHistoryEntry), cell.value.toString(),
new String[0], 100);
impactedColumns.add(i);
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset &&
map.containsKey(recon.judgmentHistoryEntry)) {
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset
&& map.containsKey(recon.judgmentHistoryEntry)) {
recon.judgment = Recon.Judgment.New;
recon.match = null;
impactedColumns.add(i);
@ -100,35 +122,36 @@ public class NewItemLibrary {
}
}
// Update reconciliation statistics for impacted columns
for(Integer colId : impactedColumns) {
for (Integer colId : impactedColumns) {
Column column = project.columnModel.getColumnByCellIndex(colId);
column.setReconStats(ReconStats.create(project, colId));
}
}
/**
* Getter, only meant to be used by Jackson
*
* @return the underlying map
*/
@JsonProperty("qidMap")
public Map<Long, String> getQidMap() {
return map;
}
@Override
public boolean equals(Object other) {
if(other == null || !NewItemLibrary.class.isInstance(other)) {
if (other == null || !NewItemLibrary.class.isInstance(other)) {
return false;
}
NewItemLibrary otherLibrary = (NewItemLibrary)other;
NewItemLibrary otherLibrary = (NewItemLibrary) other;
return map.equals(otherLibrary.getQidMap());
}
@Override
public int hashCode() {
return map.hashCode();
}
@Override
public String toString() {
return map.toString();

90
extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java
View File

@ -1,46 +1,64 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.helpers.DatamodelConverter;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A class that rewrites an {@link ItemUpdate},
* replacing reconciled entity id values by their concrete
* values after creation of all the new items involved.
* A class that rewrites an {@link ItemUpdate}, replacing reconciled entity id
* values by their concrete values after creation of all the new items involved.
*
* If an item has not been created yet, an {@link IllegalArgumentException}
* will be raised.
* If an item has not been created yet, an {@link IllegalArgumentException} will
* be raised.
*
* The subject is treated as a special case: it is returned unchanged.
* This is because it is guaranteed not to appear in the update (but
* it does appear in the datamodel representation as the subject is passed around
* to the Claim objects its document contains).
* The subject is treated as a special case: it is returned unchanged. This is
* because it is guaranteed not to appear in the update (but it does appear in
* the datamodel representation as the subject is passed around to the Claim
* objects its document contains).
*
* @author Antonin Delpeuch
*
*/
public class ReconEntityRewriter extends DatamodelConverter {
private NewItemLibrary library;
private ItemIdValue subject;
/**
* Constructor. Sets up a rewriter which uses the provided library
* to look up qids of new items, and the subject (which should not be
* rewritten).
* Constructor. Sets up a rewriter which uses the provided library to look up
* qids of new items, and the subject (which should not be rewritten).
*
* @param library
* @param subject
@ -50,39 +68,35 @@ public class ReconEntityRewriter extends DatamodelConverter {
this.library = library;
this.subject = subject;
}
@Override
public ItemIdValue copy(ItemIdValue value) {
if(subject.equals(value)) {
if (subject.equals(value)) {
return value;
}
if(value instanceof ReconItemIdValue) {
ReconItemIdValue recon = (ReconItemIdValue)value;
if(recon.isNew()) {
if (value instanceof ReconItemIdValue) {
ReconItemIdValue recon = (ReconItemIdValue) value;
if (recon.isNew()) {
String newId = library.getQid(recon.getReconInternalId());
if(newId == null) {
if (newId == null) {
throw new IllegalArgumentException(
"Trying to rewrite an update where a new item was not created yet.");
}
return Datamodel.makeItemIdValue(newId,
recon.getRecon().identifierSpace);
return Datamodel.makeItemIdValue(newId, recon.getRecon().identifierSpace);
}
}
return super.copy(value);
}
public ItemUpdate rewrite(ItemUpdate update) {
Set<MonolingualTextValue> labels = update.getLabels().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream()
.map(l -> copy(l)).collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements,
deletedStatements, labels, descriptions, aliases);
Set<MonolingualTextValue> labels = update.getLabels().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream().map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream().map(l -> copy(l))
.collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream().map(l -> copy(l))
.collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

57
extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Properties;
@ -9,37 +32,37 @@ import org.json.JSONWriter;
import com.google.refine.Jsonizable;
/**
* This is just the necessary bits to store Wikidata credentials
* in OpenRefine's preference store.
* This is just the necessary bits to store Wikidata credentials in OpenRefine's
* preference store.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
class WikibaseCredentials implements Jsonizable {
private String username;
private String password;
public WikibaseCredentials() {
username = null;
password = null;
}
public WikibaseCredentials(String username, String password) {
this.username = username;
this.password = password;
}
public String getUsername() {
return username;
}
public String getPassword() {
return password;
}
public boolean isNonNull() {
return username != null && password != null && ! "null".equals(username) && ! "null".equals(password);
return username != null && password != null && !"null".equals(username) && !"null".equals(password);
}
@Override
@ -54,12 +77,10 @@ class WikibaseCredentials implements Jsonizable {
writer.value(password);
writer.endObject();
}
public static WikibaseCredentials load(JSONObject obj) throws JSONException {
return new WikibaseCredentials(
obj.getString("username"),
obj.getString("password"));
}
}
public static WikibaseCredentials load(JSONObject obj)
throws JSONException {
return new WikibaseCredentials(obj.getString("username"), obj.getString("password"));
}
}

73
extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.math.BigDecimal;
@ -14,14 +37,12 @@ import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* Prints a Wikibase value as a string as required by QuickStatements.
* Format documentation:
* https://www.wikidata.org/wiki/Help:QuickStatements
* Prints a Wikibase value as a string as required by QuickStatements. Format
* documentation: https://www.wikidata.org/wiki/Help:QuickStatements
*
* Any new entity id will be
* assumed to be the last one created, represented with "LAST". It is
* fine to do this assumption because we are working on edit batches
* previously scheduled by {@link QuickStatementsUpdateScheduler}.
* Any new entity id will be assumed to be the last one created, represented
* with "LAST". It is fine to do this assumption because we are working on edit
* batches previously scheduled by {@link QuickStatementsUpdateScheduler}.
*
* @author Antonin Delpeuch
*
@ -37,7 +58,7 @@ public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(EntityIdValue value) {
if (ReconEntityIdValue.class.isInstance(value) && ((ReconEntityIdValue)value).isNew()) {
if (ReconEntityIdValue.class.isInstance(value) && ((ReconEntityIdValue) value).isNew()) {
return "LAST";
}
return value.getId();
@ -45,19 +66,12 @@ public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(GlobeCoordinatesValue value) {
return String.format(
Locale.US,
"@%f/%f",
value.getLatitude(),
value.getLongitude());
return String.format(Locale.US, "@%f/%f", value.getLatitude(), value.getLongitude());
}
@Override
public String visit(MonolingualTextValue value) {
return String.format(
"%s:\"%s\"",
value.getLanguageCode(),
value.getText());
return String.format("%s:\"%s\"", value.getLanguageCode(), value.getText());
}
@Override
@ -66,22 +80,16 @@ public class QSValuePrinter implements ValueVisitor<String> {
String unitIri = value.getUnit();
String unitRepresentation = "", boundsRepresentation = "";
if (!unitIri.isEmpty()) {
if (!unitIri.startsWith(unitPrefix))
return null; // QuickStatements only accepts Qids as units
unitRepresentation = "U"+unitIri.substring(unitPrefix.length());
if (!unitIri.startsWith(unitPrefix)) return null; // QuickStatements only accepts Qids as units
unitRepresentation = "U" + unitIri.substring(unitPrefix.length());
}
if (value.getLowerBound() != null) {
// bounds are always null at the same time so we know they are both not null
BigDecimal lowerBound = value.getLowerBound();
BigDecimal upperBound = value.getUpperBound();
boundsRepresentation = String.format(Locale.US, "[%s,%s]",
lowerBound.toString(), upperBound.toString());
boundsRepresentation = String.format(Locale.US, "[%s,%s]", lowerBound.toString(), upperBound.toString());
}
return String.format(
Locale.US,
"%s%s%s",
value.getNumericValue().toString(),
boundsRepresentation,
return String.format(Locale.US, "%s%s%s", value.getNumericValue().toString(), boundsRepresentation,
unitRepresentation);
}
@ -92,14 +100,7 @@ public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(TimeValue value) {
return String.format(
"+%04d-%02d-%02dT%02d:%02d:%02dZ/%d",
value.getYear(),
value.getMonth(),
value.getDay(),
value.getHour(),
value.getMinute(),
value.getSecond(),
value.getPrecision());
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
}
}

122
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.io.IOException;
@ -29,15 +52,13 @@ import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
public class QuickStatementsExporter implements WriterExporter {
final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
public static final String impossibleSchedulingErrorMessage =
"This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage =
"No schema was provided. You need to align your project with Wikidata first.";
public QuickStatementsExporter(){
public static final String impossibleSchedulingErrorMessage = "This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage = "No schema was provided. You need to align your project with Wikidata first.";
public QuickStatementsExporter() {
}
@Override
public String getContentType() {
return "text/plain";
@ -53,41 +74,45 @@ public class QuickStatementsExporter implements WriterExporter {
translateSchema(project, engine, schema, writer);
}
}
/**
* Exports a project and a schema to a QuickStatements file
*
* @param project
* the project to translate
* the project to translate
* @param engine
* the engine used for evaluation of the edits
* the engine used for evaluation of the edits
* @param schema
* the WikibaseSchema used for translation of tabular data to edits
* the WikibaseSchema used for translation of tabular data to edits
* @param writer
* the writer to which the QS should be written
* the writer to which the QS should be written
* @throws IOException
*/
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer) throws IOException {
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer)
throws IOException {
List<ItemUpdate> items = schema.evaluate(project, engine);
translateItemList(items, writer);
}
public void translateItemList(List<ItemUpdate> updates, Writer writer) throws IOException {
public void translateItemList(List<ItemUpdate> updates, Writer writer)
throws IOException {
QuickStatementsUpdateScheduler scheduler = new QuickStatementsUpdateScheduler();
try {
List<ItemUpdate> scheduled = scheduler.schedule(updates);
for (ItemUpdate item : scheduled) {
translateItem(item, writer);
}
} catch(ImpossibleSchedulingException e) {
} catch (ImpossibleSchedulingException e) {
writer.write(impossibleSchedulingErrorMessage);
}
}
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id, Writer writer) throws IOException {
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id,
Writer writer)
throws IOException {
for (MonolingualTextValue value : values) {
writer.write(qid+"\t");
writer.write(qid + "\t");
writer.write(prefix);
writer.write(value.getLanguageCode());
writer.write("\t\"");
@ -95,19 +120,20 @@ public class QuickStatementsExporter implements WriterExporter {
writer.write("\"\n");
}
}
protected void translateItem(ItemUpdate item, Writer writer) throws IOException {
protected void translateItem(ItemUpdate item, Writer writer)
throws IOException {
String qid = item.getItemId().getId();
if (item.isNew()) {
writer.write("CREATE\n");
qid = "LAST";
item = item.normalizeLabelsAndAliases();
}
translateNameDescr(qid, item.getLabels(), "L", item.getItemId(), writer);
translateNameDescr(qid, item.getDescriptions(), "D", item.getItemId(), writer);
translateNameDescr(qid, item.getAliases(), "A", item.getItemId(), writer);
for (Statement s : item.getAddedStatements()) {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), true, writer);
}
@ -115,38 +141,41 @@ public class QuickStatementsExporter implements WriterExporter {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), false, writer);
}
}
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer) throws IOException {
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer)
throws IOException {
Claim claim = statement.getClaim();
Value val = claim.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String targetValue = val.accept(vv);
if (targetValue != null) {
if (! add) {
writer.write("- ");
}
writer.write(qid + "\t" + pid + "\t" + targetValue);
for(SnakGroup q : claim.getQualifiers()) {
translateSnakGroup(q, false, writer);
}
for(Reference r : statement.getReferences()) {
for(SnakGroup g : r.getSnakGroups()) {
translateSnakGroup(g, true, writer);
}
break; // QS only supports one reference
}
writer.write("\n");
if (!add) {
writer.write("- ");
}
writer.write(qid + "\t" + pid + "\t" + targetValue);
for (SnakGroup q : claim.getQualifiers()) {
translateSnakGroup(q, false, writer);
}
for (Reference r : statement.getReferences()) {
for (SnakGroup g : r.getSnakGroups()) {
translateSnakGroup(g, true, writer);
}
break; // QS only supports one reference
}
writer.write("\n");
}
}
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer) throws IOException {
for(Snak s : sg.getSnaks()) {
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer)
throws IOException {
for (Snak s : sg.getSnaks()) {
translateSnak(s, reference, writer);
}
}
protected void translateSnak(Snak s, boolean reference, Writer writer) throws IOException {
protected void translateSnak(Snak s, boolean reference, Writer writer)
throws IOException {
String pid = s.getPropertyId().getId();
if (reference) {
pid = pid.replace('P', 'S');
@ -154,10 +183,9 @@ public class QuickStatementsExporter implements WriterExporter {
Value val = s.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String valStr = val.accept(vv);
if(valStr != null) {
if (valStr != null) {
writer.write("\t" + pid + "\t" + valStr);
}
}
}

134
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java
View File

@ -1,44 +1,48 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import org.openrefine.wikidata.editing.EditBatchProcessor;
import org.openrefine.wikidata.editing.NewItemLibrary;
import org.openrefine.wikidata.editing.ReconEntityRewriter;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.UpdateScheduler;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.util.WebResourceFetcherImpl;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.TermStatementUpdate;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -53,22 +57,19 @@ import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
import com.google.refine.util.Pool;
public class PerformWikibaseEditsOperation extends EngineDependentOperation {
static final Logger logger = LoggerFactory
.getLogger(PerformWikibaseEditsOperation.class);
static final Logger logger = LoggerFactory.getLogger(PerformWikibaseEditsOperation.class);
private String summary;
public PerformWikibaseEditsOperation(
JSONObject engineConfig,
String summary) {
public PerformWikibaseEditsOperation(JSONObject engineConfig, String summary) {
super(engineConfig);
this.summary = summary;
// getEngine(request, project);
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
@ -76,12 +77,9 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
if (obj.has("summary")) {
summary = obj.getString("summary");
}
return new PerformWikibaseEditsOperation(
engineConfig,
summary);
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
@ -96,26 +94,22 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
writer.value(getEngineConfig());
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Peform edits on Wikidata";
}
@Override
public Process createProcess(Project project, Properties options) throws Exception {
return new PerformEditsProcess(
project,
createEngine(project),
getBriefDescription(project),
summary
);
public Process createProcess(Project project, Properties options)
throws Exception {
return new PerformEditsProcess(project, createEngine(project), getBriefDescription(project), summary);
}
static public class PerformWikibaseEditsChange implements Change {
private NewItemLibrary newItemLibrary;
public PerformWikibaseEditsChange(NewItemLibrary library) {
newItemLibrary = library;
}
@ -128,7 +122,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
@Override
public void revert(Project project) {
// this does not do anything on Wikibase side -
// this does not do anything on Wikibase side -
// (we don't revert changes on Wikidata either)
newItemLibrary.updateReconciledCells(project, true);
}
@ -139,11 +133,11 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
if (newItemLibrary != null) {
writer.write("newItems=");
ObjectMapper mapper = new ObjectMapper();
writer.write(mapper.writeValueAsString(newItemLibrary)+"\n");
writer.write(mapper.writeValueAsString(newItemLibrary) + "\n");
}
writer.write("/ec/\n"); // end of change
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
NewItemLibrary library = new NewItemLibrary();
@ -152,7 +146,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("newItems".equals(field)) {
ObjectMapper mapper = new ObjectMapper();
library = mapper.readValue(value, NewItemLibrary.class);
@ -160,19 +154,18 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
}
return new PerformWikibaseEditsChange(library);
}
}
public class PerformEditsProcess extends LongRunningProcess implements Runnable {
protected Project _project;
protected Engine _engine;
protected WikibaseSchema _schema;
protected String _summary;
protected final long _historyEntryID;
protected PerformEditsProcess(Project project,
Engine engine, String description, String summary) {
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
super(description);
this._project = project;
this._engine = engine;
@ -183,7 +176,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
@Override
public void run() {
WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension");
ConnectionManager manager = ConnectionManager.getInstance();
if (!manager.isLoggedIn()) {
@ -193,21 +186,21 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, _schema.getBaseIri());
WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseIri());
// Evaluate the schema
List<ItemUpdate> itemDocuments = _schema.evaluate(_project, _engine);
// Prepare the edits
NewItemLibrary newItemLibrary = new NewItemLibrary();
EditBatchProcessor processor = new EditBatchProcessor(wbdf,
wbde, itemDocuments, newItemLibrary, _summary, 50);
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, _summary,
50);
// Perform edits
logger.info("Performing edits");
while(processor.remainingEdits() > 0) {
while (processor.remainingEdits() > 0) {
try {
processor.performEdit();
} catch(InterruptedException e) {
} catch (InterruptedException e) {
_canceled = true;
}
_progress = processor.progress();
@ -215,20 +208,15 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
break;
}
}
_progress = 100;
if (!_canceled) {
Change change = new PerformWikibaseEditsChange(newItemLibrary);
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
PerformWikibaseEditsOperation.this,
change
);
HistoryEntry historyEntry = new HistoryEntry(_historyEntryID, _project, _description,
PerformWikibaseEditsOperation.this, change);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}

77
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java
View File

@ -1,24 +1,38 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WbItemConstant;
import org.openrefine.wikidata.schema.WbItemDocumentExpr;
import org.openrefine.wikidata.schema.WbNameDescExpr;
import org.openrefine.wikidata.schema.WbStatementGroupExpr;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -33,14 +47,12 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj
.getJSONObject("schema")));
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
@ -62,32 +74,32 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
}
@Override
protected HistoryEntry createHistoryEntry(Project project,
long historyEntryID) throws Exception {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID)
throws Exception {
String description = "Save Wikibase schema skeleton";
Change change = new WikibaseSchemaChange(_schema);
return new HistoryEntry(historyEntryID, project, description,
SaveWikibaseSchemaOperation.this, change);
return new HistoryEntry(historyEntryID, project, description, SaveWikibaseSchemaOperation.this, change);
}
static public class WikibaseSchemaChange implements Change {
final protected WikibaseSchema _newSchema;
protected WikibaseSchema _oldSchema = null;
public final static String overlayModelKey = "wikibaseSchema";
public WikibaseSchemaChange(WikibaseSchema newSchema) {
_newSchema = newSchema;
}
public void apply(Project project) {
synchronized (project) {
_oldSchema = (WikibaseSchema) project.overlayModels.get(overlayModelKey);
project.overlayModels.put(overlayModelKey, _newSchema);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldSchema == null) {
@ -97,8 +109,9 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
}
}
}
public void save(Writer writer, Properties options) throws IOException {
public void save(Writer writer, Properties options)
throws IOException {
writer.write("newSchema=");
writeWikibaseSchema(_newSchema, writer);
writer.write('\n');
@ -107,33 +120,31 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
WikibaseSchema oldSchema = null;
WikibaseSchema newSchema = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
WikibaseSchemaChange change = new WikibaseSchemaChange(newSchema);
change._oldSchema = oldSchema;
return change;
}
static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer)
throws IOException {
if (s != null) {

42
extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java
View File

@ -1,4 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Set;
@ -14,16 +36,20 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public interface ConstraintFetcher {
/**
* Retrieves the regular expression for formatting a property, or null if
* there is no such constraint
* Retrieves the regular expression for formatting a property, or null if there
* is no such constraint
*
* @param pid
* @return the expression of a regular expression which should be compatible with java.util.regex
* @return the expression of a regular expression which should be compatible
* with java.util.regex
*/
String getFormatRegex(PropertyIdValue pid);
/**
* Retrieves the property that is the inverse of a given property
* @param pid: the property to retrieve the inverse for
*
* @param pid:
* the property to retrieve the inverse for
* @return the pid of the inverse property
*/
PropertyIdValue getInversePid(PropertyIdValue pid);
@ -44,12 +70,14 @@ public interface ConstraintFetcher {
boolean isForReferencesOnly(PropertyIdValue pid);
/**
* Get the list of allowed qualifiers (as property ids) for this property (null if any)
* Get the list of allowed qualifiers (as property ids) for this property (null
* if any)
*/
Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid);
/**
* Get the list of mandatory qualifiers (as property ids) for this property (null if any)
* Get the list of mandatory qualifiers (as property ids) for this property
* (null if any)
*/
Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid);

47
extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
@ -18,8 +41,6 @@ import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.WhitespaceScrutinizer;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.UpdateScheduler;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -30,15 +51,16 @@ import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
*
*/
public class EditInspector {
private Map<String, EditScrutinizer> scrutinizers;
private QAWarningStore warningStore;
private ConstraintFetcher fetcher;
public EditInspector(QAWarningStore warningStore) {
this.scrutinizers = new HashMap<>();
this.fetcher = new WikidataConstraintFetcher();
this.warningStore = warningStore;
// Register all known scrutinizers here
register(new NewItemScrutinizer());
register(new FormatScrutinizer());
@ -52,9 +74,10 @@ public class EditInspector {
register(new NoEditsMadeScrutinizer());
register(new WhitespaceScrutinizer());
}
/**
* Adds a new scrutinizer to the inspector
*
* @param scrutinizer
*/
public void register(EditScrutinizer scrutinizer) {
@ -63,11 +86,11 @@ public class EditInspector {
scrutinizer.setStore(warningStore);
scrutinizer.setFetcher(fetcher);
}
/**
* Inspect a batch of edits with the registered scrutinizers
* @param editBatch
*
* @param editBatch
*/
public void inspect(List<ItemUpdate> editBatch) {
// First, schedule them with some scheduler,
@ -75,16 +98,14 @@ public class EditInspector {
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
editBatch = scheduler.schedule(editBatch);
Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
List<ItemUpdate> mergedUpdates = updates.values().stream().collect(Collectors.toList());
for(EditScrutinizer scrutinizer : scrutinizers.values()) {
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.scrutinize(mergedUpdates);
}
if (warningStore.getNbWarnings() == 0) {
warningStore.addWarning(new QAWarning(
"no-issue-detected", null, QAWarning.Severity.INFO, 0));
warningStore.addWarning(new QAWarning("no-issue-detected", null, QAWarning.Severity.INFO, 0));
}
}
}

102
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarning.java
View File

@ -1,47 +1,69 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to represent a QA warning emitted by the Wikidata schema
* This could probably be reused at a broader scale, for instance for
* Data Package validation.
* A class to represent a QA warning emitted by the Wikidata schema This could
* probably be reused at a broader scale, for instance for Data Package
* validation.
*
* @author Antonin Delpeuch
*
*/
public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning> {
public enum Severity {
INFO, // We just report something to the user but it is probably fine
WARNING, // Edits that look wrong but in some cases they are actually fine
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we might want to allow it
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we
// might want to allow it
CRITICAL, // We should never edit if there is a critical issue
}
/// The type of QA warning emitted
private final String type;
// The key for aggregation of other QA warnings together - this specializes the id
// The key for aggregation of other QA warnings together - this specializes the
// id
private final String bucketId;
// The severity of the issue
private final Severity severity;
// The number of times this issue was found
private final int count;
// Other details about the warning, that can be displayed to the user
private final Map<String,Object> properties;
private final Map<String, Object> properties;
public QAWarning(String type, String bucketId, Severity severity, int count) {
Validate.notNull(type);
this.type = type;
@ -51,7 +73,7 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
this.count = count;
this.properties = new HashMap<>();
}
/**
* @return the full key for aggregation of QA warnings
*/
@ -63,62 +85,64 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
return this.type;
}
}
/**
* Aggregates another QA warning of the same aggregation id.
*
* @param other
*/
public QAWarning aggregate(QAWarning other) {
assert other.getAggregationId().equals(getAggregationId());
int newCount = count+other.getCount();
int newCount = count + other.getCount();
Severity newSeverity = severity;
if (other.getSeverity().compareTo(severity) > 0) {
newSeverity = other.getSeverity();
}
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity,
newCount);
for(Entry<String,Object> entry : properties.entrySet()) {
merged.setProperty(entry.getKey(),entry.getValue());
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity, newCount);
for (Entry<String, Object> entry : properties.entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
for(Entry<String,Object> entry : other.getProperties().entrySet()) {
merged.setProperty(entry.getKey(),entry.getValue());
for (Entry<String, Object> entry : other.getProperties().entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
return merged;
}
/**
* Sets a property of the QA warning, to be used by the front-end
* for display.
* @param key: the name of the property
* @param value should be Jackson-serializable
* Sets a property of the QA warning, to be used by the front-end for display.
*
* @param key:
* the name of the property
* @param value
* should be Jackson-serializable
*/
public void setProperty(String key, Object value) {
this.properties.put(key, value);
}
@JsonProperty("type")
public String getType() {
return type;
}
@JsonProperty("bucketId")
public String getBucketId() {
return bucketId;
}
@JsonProperty("severity")
public Severity getSeverity() {
return severity;
}
@JsonProperty("count")
public int getCount() {
return count;
}
@JsonProperty("properties")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Map<String,Object> getProperties() {
public Map<String, Object> getProperties() {
return properties;
}
@ -127,19 +151,17 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
*/
@Override
public int compareTo(QAWarning other) {
return - severity.compareTo(other.getSeverity());
return -severity.compareTo(other.getSeverity());
}
@Override
public boolean equals(Object other) {
if (other == null || !QAWarning.class.isInstance(other)) {
return false;
}
QAWarning otherWarning = (QAWarning)other;
return type.equals(otherWarning.getType()) &&
bucketId.equals(otherWarning.getBucketId()) &&
severity.equals(otherWarning.getSeverity()) &&
count == otherWarning.getCount() &&
properties.equals(otherWarning.getProperties());
QAWarning otherWarning = (QAWarning) other;
return type.equals(otherWarning.getType()) && bucketId.equals(otherWarning.getBucketId())
&& severity.equals(otherWarning.getSeverity()) && count == otherWarning.getCount()
&& properties.equals(otherWarning.getProperties());
}
}

38
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarningStore.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
@ -15,21 +38,22 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* @author Antonin Delpeuch
*/
public class QAWarningStore {
@JsonIgnore
private Map<String, QAWarning> map;
@JsonIgnore
private QAWarning.Severity maxSeverity;
@JsonIgnore
private int totalWarnings;
public QAWarningStore() {
this.map = new HashMap<>();
this.maxSeverity = QAWarning.Severity.INFO;
}
/**
* Stores a warning, aggregating it with any existing
* Stores a warning, aggregating it with any existing
*
* @param warning
*/
public void addWarning(QAWarning warning) {
@ -46,7 +70,7 @@ public class QAWarningStore {
map.put(aggregationKey, warning);
}
}
/**
* Returns the list of aggregated warnings, ordered by decreasing severity
*/
@ -56,7 +80,7 @@ public class QAWarningStore {
Collections.sort(result);
return result;
}
/**
* Returns the maximum severity of the stored warnings (INFO if empty)
*/
@ -64,7 +88,7 @@ public class QAWarningStore {
public QAWarning.Severity getMaxSeverity() {
return maxSeverity;
}
/**
* Returns the total number of warnings
*/

135
extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java
View File

@ -1,16 +1,36 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openrefine.wikidata.utils.EntityCache;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
@ -21,119 +41,123 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* This class provides an abstraction over the way constraint
* definitions are stored in Wikidata.
* This class provides an abstraction over the way constraint definitions are
* stored in Wikidata.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WikidataConstraintFetcher implements ConstraintFetcher {
public static String WIKIDATA_CONSTRAINT_PID = "P2302";
public static String FORMAT_CONSTRAINT_QID = "Q21502404";
public static String FORMAT_REGEX_PID = "P1793";
public static String INVERSE_CONSTRAINT_QID = "Q21510855";
public static String INVERSE_PROPERTY_PID = "P2306";
public static String USED_ONLY_AS_VALUES_CONSTRAINT_QID = "Q21528958";
public static String USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID = "Q21510863";
public static String USED_ONLY_AS_REFERENCE_CONSTRAINT_QID = "Q21528959";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_QID = "Q21510851";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_QID = "Q21510856";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String SINGLE_VALUE_CONSTRAINT_QID = "Q19474404";
public static String DISTINCT_VALUES_CONSTRAINT_QID = "Q21502410";
// The following constraints still need to be implemented:
public static String TYPE_CONSTRAINT_QID = "Q21503250";
@Override
public String getFormatRegex(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, FORMAT_CONSTRAINT_QID);
if (specs != null) {
List<Value> regexes = findValues(specs, FORMAT_REGEX_PID);
if (! regexes.isEmpty()) {
return ((StringValue)regexes.get(0)).getString();
if (!regexes.isEmpty()) {
return ((StringValue) regexes.get(0)).getString();
}
}
return null;
}
@Override
public PropertyIdValue getInversePid(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, INVERSE_CONSTRAINT_QID);
if(specs != null) {
if (specs != null) {
List<Value> inverses = findValues(specs, INVERSE_PROPERTY_PID);
if (! inverses.isEmpty()) {
return (PropertyIdValue)inverses.get(0);
if (!inverses.isEmpty()) {
return (PropertyIdValue) inverses.get(0);
}
}
return null;
}
@Override
public boolean isForValuesOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_VALUES_CONSTRAINT_QID) != null;
}
@Override
public boolean isForQualifiersOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID) != null;
return getSingleConstraint(pid, USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID) != null;
}
@Override
public boolean isForReferencesOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_REFERENCE_CONSTRAINT_QID) != null;
}
@Override
public Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, ALLOWED_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, ALLOWED_QUALIFIERS_CONSTRAINT_PID);
return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet());
}
return null;
}
@Override
public Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, MANDATORY_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, MANDATORY_QUALIFIERS_CONSTRAINT_PID);
return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet());
}
return null;
}
@Override
public boolean hasSingleValue(PropertyIdValue pid) {
return getSingleConstraint(pid, SINGLE_VALUE_CONSTRAINT_QID) != null;
}
@Override
public boolean hasDistinctValues(PropertyIdValue pid) {
return getSingleConstraint(pid, DISTINCT_VALUES_CONSTRAINT_QID) != null;
}
/**
* Returns a single constraint for a particular type and a property, or null
* if there is no such constraint
* @param pid: the property to retrieve the constraints for
* @param qid: the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not exist
* Returns a single constraint for a particular type and a property, or null if
* there is no such constraint
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not
* exist
*/
protected List<SnakGroup> getSingleConstraint(PropertyIdValue pid, String qid) {
Statement statement = getConstraintsByType(pid, qid).findFirst().orElse(null);
@ -142,23 +166,27 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
}
return null;
}
/**
* Gets the list of constraints of a particular type for a property
* @param pid: the property to retrieve the constraints for
* @param qid: the type of the constraints
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the stream of matching constraint statements
*/
protected Stream<Statement> getConstraintsByType(PropertyIdValue pid, String qid) {
Stream<Statement> allConstraints = getConstraintStatements(pid)
.stream()
Stream<Statement> allConstraints = getConstraintStatements(pid).stream()
.filter(s -> ((EntityIdValue) s.getValue()).getId().equals(qid));
return allConstraints;
}
/**
* Gets all the constraint statements for a given property
* @param pid : the id of the property to retrieve the constraints for
*
* @param pid
* : the id of the property to retrieve the constraints for
* @return the list of constraint statements
*/
protected List<Statement> getConstraintStatements(PropertyIdValue pid) {
@ -170,16 +198,19 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
return new ArrayList<Statement>();
}
}
/**
* Returns the values of a given property in qualifiers
* @param groups: the qualifiers
* @param pid: the property to filter on
*
* @param groups:
* the qualifiers
* @param pid:
* the property to filter on
* @return
*/
protected List<Value> findValues(List<SnakGroup> groups, String pid) {
List<Value> results = new ArrayList<>();
for(SnakGroup group : groups) {
for (SnakGroup group : groups) {
if (group.getProperty().getId().equals(pid)) {
for (Snak snak : group.getSnaks())
results.add(snak.getValue());

41
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java
View File

@ -1,9 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -12,18 +33,18 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for properties using the same value
* on different items.
* A scrutinizer that checks for properties using the same value on different
* items.
*
* @author Antonin Delpeuch
*
*/
public class DistinctValuesScrutinizer extends StatementScrutinizer {
public final static String type = "identical-values-for-distinct-valued-property";
private Map<PropertyIdValue, Map<Value, EntityIdValue>> _seenValues;
public DistinctValuesScrutinizer() {
_seenValues = new HashMap<>();
}
@ -40,11 +61,7 @@ public class DistinctValuesScrutinizer extends StatementScrutinizer {
}
if (seen.containsKey(mainSnakValue)) {
EntityIdValue otherId = seen.get(mainSnakValue);
QAWarning issue = new QAWarning(
type,
pid.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("item1_entity", entityId);
issue.setProperty("item2_entity", otherId);

52
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java
View File

@ -1,8 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
import org.openrefine.wikidata.qa.WikidataConstraintFetcher;
import org.openrefine.wikidata.qa.ConstraintFetcher;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarning.Severity;
@ -15,64 +37,70 @@ import org.openrefine.wikidata.updates.ItemUpdate;
* @author Antonin Delpeuch
*/
public abstract class EditScrutinizer {
protected QAWarningStore _store;
protected ConstraintFetcher _fetcher;
public EditScrutinizer() {
_fetcher = null;
_store = null;
}
public void setStore(QAWarningStore store) {
_store = store;
}
public void setFetcher(ConstraintFetcher fetcher) {
_fetcher = fetcher;
}
/**
* Reads the candidate edits and emits warnings in the store
* @param edit: the list of ItemUpdates to scrutinize
*
* @param edit:
* the list of ItemUpdates to scrutinize
*/
public abstract void scrutinize(List<ItemUpdate> edit);
protected void addIssue(QAWarning warning) {
_store.addWarning(warning);
}
protected void addIssue(String type, String aggregationId, Severity severity, int count) {
addIssue(new QAWarning(type, aggregationId, severity, count));
}
/**
* Helper to be used by subclasses to emit simple INFO warnings
*
* @param warning
*/
protected void info(String type) {
addIssue(type, null, QAWarning.Severity.INFO, 1);
}
/**
* Helper to be used by subclasses to emit simple warnings
*
* @param warning
*/
protected void warning(String type) {
addIssue(type, null, QAWarning.Severity.WARNING, 1);
}
/**
* Helper to be used by subclasses to emit simple important warnings
*
* @param warning
*/
protected void important(String type) {
addIssue(type, null, QAWarning.Severity.IMPORTANT, 1);
}
/**
* Helper to be used by subclasses to emit simple critical warnings
*
* @param warning
*/
protected void critical(String type) {

55
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java
View File

@ -1,8 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
@ -12,31 +34,32 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
/**
* A scrutinizer that detects incorrect formats in text values
* (mostly identifiers).
* A scrutinizer that detects incorrect formats in text values (mostly
* identifiers).
*
* @author Antonin Delpeuch
*
*/
public class FormatScrutinizer extends SnakScrutinizer {
public static final String type = "add-statements-with-invalid-format";
private Map<PropertyIdValue, Pattern> _patterns;
public FormatScrutinizer() {
_patterns = new HashMap<>();
}
/**
* Loads the regex for a property and compiles it to a pattern
* (this is cached upstream, plus we are doing it only once per
* property and batch).
* @param pid the id of the property to fetch the constraints for
* Loads the regex for a property and compiles it to a pattern (this is cached
* upstream, plus we are doing it only once per property and batch).
*
* @param pid
* the id of the property to fetch the constraints for
* @return
*/
protected Pattern getPattern(PropertyIdValue pid) {
if(_patterns.containsKey(pid)) {
if (_patterns.containsKey(pid)) {
return _patterns.get(pid);
} else {
String regex = _fetcher.getFormatRegex(pid);
@ -51,7 +74,7 @@ public class FormatScrutinizer extends SnakScrutinizer {
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if(StringValue.class.isInstance(snak.getValue())) {
if (StringValue.class.isInstance(snak.getValue())) {
String value = ((StringValue) snak.getValue()).getString();
PropertyIdValue pid = snak.getPropertyId();
Pattern pattern = getPattern(pid);
@ -60,11 +83,7 @@ public class FormatScrutinizer extends SnakScrutinizer {
}
if (!pattern.matcher(value).matches()) {
if (added) {
QAWarning issue = new QAWarning(
type,
pid.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("regex", pattern.toString());
issue.setProperty("example_value", value);

57
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -14,48 +37,47 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for missing inverse statements in
* edit batches.
* A scrutinizer that checks for missing inverse statements in edit batches.
*
* @author Antonin Delpeuch
*
*/
public class InverseConstraintScrutinizer extends StatementScrutinizer {
public static final String type = "missing-inverse-statements";
private Map<PropertyIdValue, PropertyIdValue> _inverse;
private Map<PropertyIdValue, Map<EntityIdValue, Set<EntityIdValue> >> _statements;
private Map<PropertyIdValue, Map<EntityIdValue, Set<EntityIdValue>>> _statements;
public InverseConstraintScrutinizer() {
_inverse = new HashMap<>();
_statements = new HashMap<>();
}
protected PropertyIdValue getInverseConstraint(PropertyIdValue pid) {
if (_inverse.containsKey(pid)) {
return _inverse.get(pid);
} else {
PropertyIdValue inversePid = _fetcher.getInversePid(pid);
_inverse.put(pid, inversePid);
_statements.put(pid, new HashMap<EntityIdValue,Set<EntityIdValue>>());
_statements.put(pid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
// We are doing this check because we do not have any guarantee that
// the inverse constraints are consistent on Wikidata.
if (inversePid != null && !_inverse.containsKey(inversePid)) {
_inverse.put(inversePid, pid);
_statements.put(inversePid, new HashMap<EntityIdValue,Set<EntityIdValue>>());
_statements.put(inversePid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
}
return inversePid;
}
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if (!added) {
return; // TODO support for deleted statements
}
Value mainSnakValue = statement.getClaim().getMainSnak().getValue();
if (ItemIdValue.class.isInstance(mainSnakValue)) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
@ -71,24 +93,21 @@ public class InverseConstraintScrutinizer extends StatementScrutinizer {
}
}
}
@Override
public void batchIsFinished() {
// For each pair of inverse properties (in each direction)
for(Entry<PropertyIdValue,PropertyIdValue> propertyPair : _inverse.entrySet()) {
for (Entry<PropertyIdValue, PropertyIdValue> propertyPair : _inverse.entrySet()) {
// Get the statements made for the first
PropertyIdValue ourProperty = propertyPair.getKey();
for(Entry<EntityIdValue, Set<EntityIdValue>> itemLinks : _statements.get(ourProperty).entrySet()) {
for (Entry<EntityIdValue, Set<EntityIdValue>> itemLinks : _statements.get(ourProperty).entrySet()) {
// For each outgoing link
for(EntityIdValue idValue : itemLinks.getValue()) {
for (EntityIdValue idValue : itemLinks.getValue()) {
// Check that they are in the statements made for the second
PropertyIdValue missingProperty = propertyPair.getValue();
Set<EntityIdValue> reciprocalLinks = _statements.get(missingProperty).get(idValue);
if (reciprocalLinks == null || !reciprocalLinks.contains(itemLinks.getKey())) {
QAWarning issue = new QAWarning(type,
ourProperty.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, ourProperty.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("added_property_entity", ourProperty);
issue.setProperty("inverse_property_entity", missingProperty);
issue.setProperty("source_entity", itemLinks.getKey());

38
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
@ -8,8 +31,8 @@ public abstract class ItemUpdateScrutinizer extends EditScrutinizer {
@Override
public void scrutinize(List<ItemUpdate> edit) {
for(ItemUpdate update : edit) {
if(!update.isNull()) {
for (ItemUpdate update : edit) {
if (!update.isNull()) {
scrutinize(update);
}
}
@ -17,15 +40,16 @@ public abstract class ItemUpdateScrutinizer extends EditScrutinizer {
}
/**
* Method to be overridden by subclasses to scrutinize
* an individual item update.
* Method to be overridden by subclasses to scrutinize an individual item
* update.
*
* @param update
*/
public abstract void scrutinize(ItemUpdate update);
/**
* Method to be overridden by subclasses to emit warnings
* once a batch has been completely analyzed.
* Method to be overridden by subclasses to emit warnings once a batch has been
* completely analyzed.
*/
public void batchIsFinished() {
;

63
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
@ -10,7 +33,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
* @author Antonin Delpeuch
*/
public class NewItemScrutinizer extends ItemUpdateScrutinizer {
public static final String noLabelType = "new-item-without-labels-or-aliases";
public static final String noDescType = "new-item-without-descriptions";
public static final String deletedStatementsType = "new-item-with-deleted-statements";
@ -21,40 +44,28 @@ public class NewItemScrutinizer extends ItemUpdateScrutinizer {
public void scrutinize(ItemUpdate update) {
if (update.isNew()) {
info(newItemType);
if (update.getLabels().isEmpty() && update.getAliases().isEmpty()) {
QAWarning issue = new QAWarning(
noLabelType,
null,
QAWarning.Severity.CRITICAL,
1);
QAWarning issue = new QAWarning(noLabelType, null, QAWarning.Severity.CRITICAL, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (update.getDescriptions().isEmpty()) {
QAWarning issue = new QAWarning(
noDescType,
null,
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(noDescType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (!update.getDeletedStatements().isEmpty()) {
QAWarning issue = new QAWarning(
deletedStatementsType,
null,
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(deletedStatementsType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
// Try to find a "instance of" or "subclass of" claim
boolean typeFound = false;
for(StatementGroup group : update.getAddedStatementGroups()) {
for (StatementGroup group : update.getAddedStatementGroups()) {
String pid = group.getProperty().getId();
if ("P31".equals(pid) || "P279".equals(pid)) {
typeFound = true;
@ -62,13 +73,9 @@ public class NewItemScrutinizer extends ItemUpdateScrutinizer {
}
}
if (!typeFound) {
QAWarning issue = new QAWarning(
noTypeType,
null,
QAWarning.Severity.WARNING,
1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
QAWarning issue = new QAWarning(noTypeType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
}
}

28
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java
View File

@ -1,17 +1,39 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
import org.openrefine.wikidata.updates.ItemUpdate;
public class NoEditsMadeScrutinizer extends EditScrutinizer {
public static final String type = "no-edit-generated";
@Override
public void scrutinize(List<ItemUpdate> edit) {
if(edit.stream().allMatch(e -> e.isNull())) {
if (edit.stream().allMatch(e -> e.isNull())) {
info(type);
}

67
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -12,24 +35,24 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that checks the compatibility of the qualifiers
* and the property of a statement, and looks for mandatory qualifiers.
* A scrutinizer that checks the compatibility of the qualifiers and the
* property of a statement, and looks for mandatory qualifiers.
*
* @author Antonin Delpeuch
*/
public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
public static final String missingMandatoryQualifiersType = "missing-mandatory-qualifiers";
public static final String disallowedQualifiersType = "disallowed-qualifiers";
private Map<PropertyIdValue, Set<PropertyIdValue>> _allowedQualifiers;
private Map<PropertyIdValue, Set<PropertyIdValue>> _mandatoryQualifiers;
public QualifierCompatibilityScrutinizer() {
_allowedQualifiers = new HashMap<>();
_mandatoryQualifiers = new HashMap<>();
}
protected boolean qualifierIsAllowed(PropertyIdValue statementProperty, PropertyIdValue qualifierProperty) {
Set<PropertyIdValue> allowed = null;
if (_allowedQualifiers.containsKey(statementProperty)) {
@ -40,7 +63,7 @@ public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
}
return allowed == null || allowed.contains(qualifierProperty);
}
protected Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue statementProperty) {
Set<PropertyIdValue> mandatory = null;
if (_mandatoryQualifiers.containsKey(statementProperty)) {
@ -58,31 +81,25 @@ public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue statementProperty = statement.getClaim().getMainSnak().getPropertyId();
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().
stream().map(e -> e.getProperty()).collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty)
.stream().filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers
.stream().filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().stream().map(e -> e.getProperty())
.collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty).stream()
.filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers.stream()
.filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
for (PropertyIdValue missing : missingQualifiers) {
QAWarning issue = new QAWarning(
missingMandatoryQualifiersType,
statementProperty.getId()+"-"+missing.getId(),
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(missingMandatoryQualifiersType,
statementProperty.getId() + "-" + missing.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("missing_property_entity", missing);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
for (PropertyIdValue disallowed : disallowedQualifiers) {
QAWarning issue = new QAWarning(
disallowedQualifiersType,
statementProperty.getId()+"-"+disallowed.getId(),
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(disallowedQualifiersType,
statementProperty.getId() + "-" + disallowed.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("disallowed_property_entity", disallowed);
issue.setProperty("example_item_entity", entityId);

66
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedPositionScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -13,25 +36,22 @@ import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class RestrictedPositionScrutinizer extends StatementScrutinizer {
protected enum SnakPosition {
MAINSNAK,
QUALIFIER,
REFERENCE
MAINSNAK, QUALIFIER, REFERENCE
}
private Map<PropertyIdValue, SnakPosition> _restrictedPids;
private Set<PropertyIdValue> _unrestrictedPids;
public RestrictedPositionScrutinizer() {
_restrictedPids = new HashMap<>();
_unrestrictedPids = new HashSet<>();
}
SnakPosition positionRestriction(PropertyIdValue pid) {
if(_unrestrictedPids.contains(pid)) {
if (_unrestrictedPids.contains(pid)) {
return null;
}
SnakPosition restriction = _restrictedPids.get(pid);
@ -45,7 +65,7 @@ public class RestrictedPositionScrutinizer extends StatementScrutinizer {
} else if (_fetcher.isForReferencesOnly(pid)) {
restriction = SnakPosition.REFERENCE;
}
// Cache these results:
if (restriction != null) {
_restrictedPids.put(pid, restriction);
@ -55,39 +75,37 @@ public class RestrictedPositionScrutinizer extends StatementScrutinizer {
return restriction;
}
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Skip the main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, SnakPosition.MAINSNAK, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, SnakPosition.QUALIFIER, added);
// References
for(Reference ref : statement.getReferences()) {
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, SnakPosition.REFERENCE, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position, boolean added) {
while(snaks.hasNext()) {
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position,
boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, position, added);
}
}
public void scrutinize(Snak snak, EntityIdValue entityId, SnakPosition position, boolean added) {
SnakPosition restriction = positionRestriction(snak.getPropertyId());
if (restriction != null && position != restriction) {
String positionStr = position.toString().toLowerCase();
String restrictionStr = restriction.toString().toLowerCase();
QAWarning issue = new QAWarning(
"property-restricted-to-"+restrictionStr+"-found-in-"+positionStr,
snak.getPropertyId().getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning("property-restricted-to-" + restrictionStr + "-found-in-" + positionStr,
snak.getPropertyId().getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", snak.getPropertyId());
addIssue(issue);
}

35
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
@ -5,22 +28,20 @@ import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
/**
* A scrutinizer that checks for self-referential statements.
* These statements are flagged by Wikibase as suspicious.
* A scrutinizer that checks for self-referential statements. These statements
* are flagged by Wikibase as suspicious.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class SelfReferentialScrutinizer extends SnakScrutinizer {
public static final String type = "self-referential-statements";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (entityId.equals(snak.getValue())) {
QAWarning issue = new QAWarning(
type, null,
QAWarning.Severity.WARNING, 1);
QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", entityId);
addIssue(issue);
}

39
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashSet;
@ -9,27 +32,25 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* For now this scrutinizer only checks for uniqueness at
* the item level (it ignores qualifiers and references).
* For now this scrutinizer only checks for uniqueness at the item level (it
* ignores qualifiers and references).
*
* @author Antonin Delpeuch
*
*/
public class SingleValueScrutinizer extends ItemUpdateScrutinizer {
public static final String type = "single-valued-property-added-more-than-once";
@Override
public void scrutinize(ItemUpdate update) {
Set<PropertyIdValue> seenSingleProperties = new HashSet<>();
for(Statement statement : update.getAddedStatements()) {
for (Statement statement : update.getAddedStatements()) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (seenSingleProperties.contains(pid)) {
QAWarning issue = new QAWarning(
type, pid.getId(),
QAWarning.Severity.WARNING, 1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);

51
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SnakScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Iterator;
@ -8,38 +31,42 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that inspects snaks individually, no matter whether they
* appear as main snaks, qualifiers or references.
* A scrutinizer that inspects snaks individually, no matter whether they appear
* as main snaks, qualifiers or references.
*
* @author Antonin Delpeuch
*
*/
public abstract class SnakScrutinizer extends StatementScrutinizer {
/**
* This is the method that subclasses should override to implement their checks.
* @param snak: the snak to inspect
* @param entityId: the item on which it is going to (dis)appear
* @param added: whether this snak is going to be added or deleted
*
* @param snak:
* the snak to inspect
* @param entityId:
* the item on which it is going to (dis)appear
* @param added:
* whether this snak is going to be added or deleted
*/
public abstract void scrutinize(Snak snak, EntityIdValue entityId, boolean added);
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, added);
// References
for(Reference ref : statement.getReferences()) {
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, boolean added) {
while(snaks.hasNext()) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, added);
}

39
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
@ -9,7 +32,7 @@ public abstract class StatementScrutinizer extends ItemUpdateScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
EntityIdValue currentEntityId = update.getItemId();
for(Statement statement : update.getAddedStatements()) {
for (Statement statement : update.getAddedStatements()) {
scrutinize(statement, currentEntityId, true);
}
for (Statement statement : update.getDeletedStatements()) {
@ -18,11 +41,15 @@ public abstract class StatementScrutinizer extends ItemUpdateScrutinizer {
}
/**
* The method that should be overridden by subclasses, implementing
* the checks on one statement
* @param statement: the statement to scrutinize
* @param entityId: the id of the entity on which this statement is made or removed
* @param added: whether this statement was added or deleted
* The method that should be overridden by subclasses, implementing the checks
* on one statement
*
* @param statement:
* the statement to scrutinize
* @param entityId:
* the id of the entity on which this statement is made or removed
* @param added:
* whether this statement was added or deleted
*/
public abstract void scrutinize(Statement statement, EntityIdValue entityId, boolean added);
}

29
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -6,16 +29,16 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer checking for unsourced statements
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class UnsourcedScrutinizer extends StatementScrutinizer {
public static final String type = "unsourced-statements";
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if(statement.getReferences().isEmpty() && added) {
if (statement.getReferences().isEmpty() && added) {
warning(type);
}
}

36
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
@ -8,22 +31,23 @@ import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that inspects the values of snaks and terms
* @author antonin
*
* @author Antonin Delpeuch
*
*/
public abstract class ValueScrutinizer extends SnakScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
super.scrutinize(update);
for(MonolingualTextValue label : update.getLabels()) {
for (MonolingualTextValue label : update.getLabels()) {
scrutinize(label);
}
for(MonolingualTextValue alias : update.getAliases()) {
for (MonolingualTextValue alias : update.getAliases()) {
scrutinize(alias);
}
for(MonolingualTextValue description : update.getDescriptions()) {
for (MonolingualTextValue description : update.getDescriptions()) {
scrutinize(description);
}
}

47
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/WhitespaceScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -17,20 +40,20 @@ import org.wikidata.wdtk.datamodel.interfaces.Value;
*
*/
public class WhitespaceScrutinizer extends ValueScrutinizer {
private Map<String,Pattern> _issuesMap;
private Map<String, Pattern> _issuesMap;
public static final String leadingWhitespaceType = "leading-whitespace";
public static final String trailingWhitespaceType = "trailing-whitespace";
public static final String duplicateWhitespaceType = "duplicate-whitespace";
public static final String nonPrintableCharsType = "non-printable-characters";
public WhitespaceScrutinizer() {
_issuesMap = new HashMap<>();
_issuesMap.put(leadingWhitespaceType, Pattern.compile("^\\s"));
_issuesMap.put(trailingWhitespaceType, Pattern.compile("\\s$"));
_issuesMap.put(duplicateWhitespaceType, Pattern.compile("\\s\\s"));
// https://stackoverflow.com/questions/14565934/regular-expression-to-remove-all-non-printable-characters
_issuesMap.put(nonPrintableCharsType, Pattern.compile("[\\x00\\x03\\x08\\x0B\\x0C\\x0E-\\x1F]"));
}
@ -38,21 +61,21 @@ public class WhitespaceScrutinizer extends ValueScrutinizer {
@Override
public void scrutinize(Value value) {
String str = null;
if(MonolingualTextValue.class.isInstance(value)) {
str = ((MonolingualTextValue)value).getText();
if (MonolingualTextValue.class.isInstance(value)) {
str = ((MonolingualTextValue) value).getText();
} else if (StringValue.class.isInstance(value)) {
str = ((StringValue)value).getString();
str = ((StringValue) value).getString();
}
if (str != null) {
for(Entry<String,Pattern> entry : _issuesMap.entrySet()) {
if(entry.getValue().matcher(str).find()) {
for (Entry<String, Pattern> entry : _issuesMap.entrySet()) {
if (entry.getValue().matcher(str).find()) {
emitWarning(entry.getKey(), str);
}
}
}
}
private void emitWarning(String type, String example) {
QAWarning warning = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_string", example);

66
extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -10,39 +33,36 @@ import com.google.refine.model.ColumnModel;
import com.google.refine.model.Row;
/**
* A class holding all the necessary information about
* the context in which a schema expression is evaluated.
* A class holding all the necessary information about the context in which a
* schema expression is evaluated.
*
* @author Antonin Delpeuch
*
*/
public class ExpressionContext {
private String baseIRI;
private int rowId;
private Row row;
private ColumnModel columnModel;
private QAWarningStore warningStore;
/**
* Builds an expression context to evaluate a schema on a row
*
* @param baseIRI
* the siteIRI of the schema
* the siteIRI of the schema
* @param rowId
* the id of the row currently visited
* the id of the row currently visited
* @param row
* the row itself
* the row itself
* @param columnModel
* lets us access cells by column name
* lets us access cells by column name
* @param warningStore
* where to store the issues encountered when
* evaluating (can be set to null if these issues should be ignored)
* where to store the issues encountered when evaluating (can be set
* to null if these issues should be ignored)
*/
public ExpressionContext(
String baseIRI,
int rowId,
Row row,
ColumnModel columnModel,
QAWarningStore warningStore) {
public ExpressionContext(String baseIRI, int rowId, Row row, ColumnModel columnModel, QAWarningStore warningStore) {
Validate.notNull(baseIRI);
this.baseIRI = baseIRI;
this.rowId = rowId;
@ -52,18 +72,18 @@ public class ExpressionContext {
this.columnModel = columnModel;
this.warningStore = warningStore;
}
public String getBaseIRI() {
return baseIRI;
}
/**
* Retrieves a cell in the current row, by column name.
* If the column does not exist, null is returned.
* Retrieves a cell in the current row, by column name. If the column does not
* exist, null is returned.
*
* @param name
* the name of the column to retrieve the cell from
* @return
* the cell
* the name of the column to retrieve the cell from
* @return the cell
*/
public Cell getCellByName(String name) {
Column column = columnModel.getColumnByName(name);
@ -74,11 +94,11 @@ public class ExpressionContext {
return null;
}
}
public int getRowId() {
return rowId;
}
public void addWarning(QAWarning warning) {
if (warningStore != null) {
warningStore.addWarning(warning);

121
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java
View File

@ -1,12 +1,35 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -19,8 +42,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
/**
* A constant for a time value, accepting a number of formats
* which determine the precision of the parsed value.
* A constant for a time value, accepting a number of formats which determine
* the precision of the parsed value.
*
* @author Antonin Delpeuch
*
@ -28,60 +51,57 @@ import com.google.common.collect.ImmutableMap;
public class WbDateConstant implements WbExpression<TimeValue> {
/**
* Map of formats accepted by the parser. Each format is associated
* to the time precision it induces (an integer according to Wikibase's data model).
* Map of formats accepted by the parser. Each format is associated to the time
* precision it induces (an integer according to Wikibase's data model).
*/
public static Map<SimpleDateFormat,Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat,Integer>builder()
.put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14)
.build();
public static Map<SimpleDateFormat, Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat, Integer> builder()
.put(new SimpleDateFormat("yyyy"), 9).put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11).put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build();
private TimeValue parsed;
private String origDatestamp;
/**
* Constructor. Used for deserialization from JSON.
* The object will be constructed even if the time cannot
* be parsed (it will evaluate to null) in {@link evaluate}.
* Constructor. Used for deserialization from JSON. The object will be
* constructed even if the time cannot be parsed (it will evaluate to null) in
* {@link evaluate}.
*
* @param origDatestamp
* the date value as a string
* the date value as a string
*/
@JsonCreator
public WbDateConstant(
@JsonProperty("value") String origDatestamp) {
public WbDateConstant(@JsonProperty("value") String origDatestamp) {
Validate.notNull(origDatestamp);
this.setOrigDatestamp(origDatestamp);
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return parsed;
}
/**
* Parses a timestamp into a Wikibase {@link TimeValue}. The
* precision is automatically inferred from the format.
* Parses a timestamp into a Wikibase {@link TimeValue}. The precision is
* automatically inferred from the format.
*
* @param datestamp
* the time to parse
* the time to parse
* @return
* @throws ParseException
* if the time cannot be parsed
* if the time cannot be parsed
*/
public static TimeValue parse(String datestamp) throws ParseException {
public static TimeValue parse(String datestamp)
throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for(Entry<SimpleDateFormat,Integer> entry : acceptedFormats.entrySet()) {
for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) {
ParsePosition position = new ParsePosition(0);
String trimmedDatestamp = datestamp.trim();
date = entry.getKey().parse(trimmedDatestamp, position);
// Ignore parses which failed or do not consume all the input
if (date != null && position.getIndex() == trimmedDatestamp.length()) {
precision = entry.getValue();
@ -94,21 +114,16 @@ public class WbDateConstant implements WbExpression<TimeValue> {
Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance();
calendar.setTime(date);
return Datamodel.makeTimeValue(
calendar.get(Calendar.YEAR),
(byte) (calendar.get(Calendar.MONTH)+1), // java starts at 0
(byte) calendar.get(Calendar.DAY_OF_MONTH),
(byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE),
(byte) calendar.get(Calendar.SECOND),
(byte) precision,
0,
1,
calendar.getTimeZone().getRawOffset()/3600000,
TimeValue.CM_GREGORIAN_PRO);
}
return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), // java
// starts
// at
// 0
(byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 1,
calendar.getTimeZone().getRawOffset() / 3600000, TimeValue.CM_GREGORIAN_PRO);
}
}
/**
* @return the original datestamp
*/
@ -120,21 +135,21 @@ public class WbDateConstant implements WbExpression<TimeValue> {
private void setOrigDatestamp(String origDatestamp) {
this.origDatestamp = origDatestamp;
try {
this.parsed = parse(origDatestamp);
} catch(ParseException e) {
throw new IllegalArgumentException("Invalid datestamp provided: "+origDatestamp);
this.parsed = parse(origDatestamp);
} catch (ParseException e) {
throw new IllegalArgumentException("Invalid datestamp provided: " + origDatestamp);
}
}
@Override
public boolean equals(Object other) {
if(other == null || !WbDateConstant.class.isInstance(other)) {
if (other == null || !WbDateConstant.class.isInstance(other)) {
return false;
}
WbDateConstant otherConstant = (WbDateConstant)other;
WbDateConstant otherConstant = (WbDateConstant) other;
return origDatestamp.equals(otherConstant.getOrigDatestamp());
}
@Override
public int hashCode() {
return origDatestamp.hashCode();

35
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -10,19 +33,19 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* An expression that represents a time value, extracted from a string.
* A number of formats are recognized, see {@link WbDateConstant} for details.
* An expression that represents a time value, extracted from a string. A number
* of formats are recognized, see {@link WbDateConstant} for details.
*
* @author Antonin Delpeuch
*
*/
public class WbDateVariable extends WbVariableExpr<TimeValue> {
@JsonCreator
public WbDateVariable() {
}
public WbDateVariable(String columnName) {
setColumnName(columnName);
}
@ -37,7 +60,7 @@ public class WbDateVariable extends WbVariableExpr<TimeValue> {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbDateVariable.class);

74
extensions/wikidata/src/org/openrefine/wikidata/schema/WbExpression.java
View File

@ -1,40 +1,60 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The base interface for all expressions, which evaluate to a
* particular type T in an ExpressionContext.
* The base interface for all expressions, which evaluate to a particular type T
* in an ExpressionContext.
*/
@JsonTypeInfo(use=JsonTypeInfo.Id.NAME,
include=JsonTypeInfo.As.PROPERTY,
property="type")
@JsonSubTypes({
@Type(value = WbStringConstant.class, name = "wbstringconstant"),
@Type(value = WbStringVariable.class, name = "wbstringvariable"),
@Type(value = WbLocationConstant.class, name = "wblocationconstant"),
@Type(value = WbLocationVariable.class, name = "wblocationvariable"),
@Type(value = WbItemConstant.class, name = "wbitemconstant"),
@Type(value = WbItemVariable.class, name = "wbitemvariable"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbDateConstant.class, name = "wbdateconstant"),
@Type(value = WbDateVariable.class, name = "wbdatevariable"),
@Type(value = WbMonolingualExpr.class, name = "wbmonolingualexpr"),
@Type(value = WbPropConstant.class, name = "wbpropconstant"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbQuantityExpr.class, name="wbquantityexpr"),
})
public interface WbExpression<T> {
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
@JsonSubTypes({ @Type(value = WbStringConstant.class, name = "wbstringconstant"),
@Type(value = WbStringVariable.class, name = "wbstringvariable"),
@Type(value = WbLocationConstant.class, name = "wblocationconstant"),
@Type(value = WbLocationVariable.class, name = "wblocationvariable"),
@Type(value = WbItemConstant.class, name = "wbitemconstant"),
@Type(value = WbItemVariable.class, name = "wbitemvariable"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbDateConstant.class, name = "wbdateconstant"),
@Type(value = WbDateVariable.class, name = "wbdatevariable"),
@Type(value = WbMonolingualExpr.class, name = "wbmonolingualexpr"),
@Type(value = WbPropConstant.class, name = "wbpropconstant"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbQuantityExpr.class, name = "wbquantityexpr"), })
public interface WbExpression<T> {
/**
* Evaluates the value expression in a given context,
* returns a Wikibase value suitable to be the target of a claim.
* Evaluates the value expression in a given context, returns a Wikibase value
* suitable to be the target of a claim.
*/
public T evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException;
public T evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException;
}

47
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -8,18 +31,15 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Represents an item that does not vary,
* it is independent of the row.
* Represents an item that does not vary, it is independent of the row.
*/
public class WbItemConstant implements WbExpression<ItemIdValue> {
private String qid;
private String label;
@JsonCreator
public WbItemConstant(
@JsonProperty("qid") String qid,
@JsonProperty("label") String label) {
public WbItemConstant(@JsonProperty("qid") String qid, @JsonProperty("label") String label) {
Validate.notNull(qid);
this.qid = qid;
Validate.notNull(label);
@ -28,10 +48,7 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
@Override
public ItemIdValue evaluate(ExpressionContext ctxt) {
return new SuggestedItemIdValue(
qid,
ctxt.getBaseIRI(),
label);
return new SuggestedItemIdValue(qid, ctxt.getBaseIRI(), label);
}
@JsonProperty("qid")
@ -43,16 +60,16 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
public String getLabel() {
return label;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbItemConstant.class.isInstance(other)) {
if (other == null || !WbItemConstant.class.isInstance(other)) {
return false;
}
WbItemConstant otherConstant = (WbItemConstant)other;
WbItemConstant otherConstant = (WbItemConstant) other;
return (qid.equals(otherConstant.getQid()) && label.equals(otherConstant.getLabel()));
}
@Override
public int hashCode() {
return qid.hashCode() + label.hashCode();

62
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.Collections;
@ -17,51 +40,51 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The representation of an item document, which can contain
* variables both for its own id and in its contents.
* The representation of an item document, which can contain variables both for
* its own id and in its contents.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use=JsonTypeInfo.Id.NONE)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpression<ItemUpdate> {
private WbExpression<? extends ItemIdValue> subject;
private List<WbNameDescExpr> nameDescs;
private List<WbStatementGroupExpr> statementGroups;
@JsonCreator
public WbItemDocumentExpr(
@JsonProperty("subject") WbExpression<? extends ItemIdValue> subjectExpr,
public WbItemDocumentExpr(@JsonProperty("subject") WbExpression<? extends ItemIdValue> subjectExpr,
@JsonProperty("nameDescs") List<WbNameDescExpr> nameDescExprs,
@JsonProperty("statementGroups") List<WbStatementGroupExpr> statementGroupExprs) {
Validate.notNull(subjectExpr);
this.subject = subjectExpr;
if(nameDescExprs == null) {
if (nameDescExprs == null) {
nameDescExprs = Collections.emptyList();
}
this.nameDescs = nameDescExprs;
if(statementGroupExprs == null) {
if (statementGroupExprs == null) {
statementGroupExprs = Collections.emptyList();
}
this.statementGroups = statementGroupExprs;
}
@Override
public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public ItemUpdate evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
ItemIdValue subjectId = getSubject().evaluate(ctxt);
ItemUpdateBuilder update = new ItemUpdateBuilder(subjectId);
for(WbStatementGroupExpr expr : getStatementGroups()) {
for (WbStatementGroupExpr expr : getStatementGroups()) {
try {
for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
for (Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
update.addStatement(s);
}
} catch (SkipSchemaExpressionException e) {
continue;
}
}
for(WbNameDescExpr expr : getNameDescs()) {
for (WbNameDescExpr expr : getNameDescs()) {
expr.contributeTo(update, ctxt);
}
return update.build();
@ -81,18 +104,17 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio
public List<WbStatementGroupExpr> getStatementGroups() {
return statementGroups;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbItemDocumentExpr.class.isInstance(other)) {
if (other == null || !WbItemDocumentExpr.class.isInstance(other)) {
return false;
}
WbItemDocumentExpr otherExpr = (WbItemDocumentExpr)other;
return subject.equals(otherExpr.getSubject()) &&
nameDescs.equals(otherExpr.getNameDescs()) &&
statementGroups.equals(otherExpr.getStatementGroups());
WbItemDocumentExpr otherExpr = (WbItemDocumentExpr) other;
return subject.equals(otherExpr.getSubject()) && nameDescs.equals(otherExpr.getNameDescs())
&& statementGroups.equals(otherExpr.getStatementGroups());
}
@Override
public int hashCode() {
return subject.hashCode() + nameDescs.hashCode() + statementGroups.hashCode();

42
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -1,6 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
@ -17,33 +39,33 @@ import com.google.refine.model.Recon.Judgment;
*
*/
public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
@JsonCreator
public WbItemVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
* the name of the column the expression should draw its value from
*/
public WbItemVariable(String columnName) {
setColumnName(columnName);
}
@Override
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException {
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (cell.recon != null
&& (Judgment.Matched.equals(cell.recon.judgment) ||
Judgment.New.equals(cell.recon.judgment))) {
&& (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) {
return new ReconItemIdValue(cell.recon, cell.value.toString());
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbItemVariable.class);

63
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -14,43 +37,41 @@ import com.fasterxml.jackson.annotation.JsonProperty;
*
*/
public class WbLanguageConstant implements WbExpression<String> {
protected String _langId;
protected String _langLabel;
@JsonCreator
public WbLanguageConstant(
@JsonProperty("id") String langId,
@JsonProperty("label") String langLabel) {
public WbLanguageConstant(@JsonProperty("id") String langId, @JsonProperty("label") String langLabel) {
_langId = normalizeLanguageCode(langId);
Validate.notNull(_langId, "A valid language code must be provided.");
Validate.notNull(langLabel);
_langLabel = langLabel;
}
/**
* Checks that a language code is valid and returns its preferred
* version (converting deprecated language codes to their better values).
* Checks that a language code is valid and returns its preferred version
* (converting deprecated language codes to their better values).
*
* @param lang
* a Wikimedia language code
* @return
* the normalized code, or null if the code is invalid.
* a Wikimedia language code
* @return the normalized code, or null if the code is invalid.
*/
public static String normalizeLanguageCode(String lang) {
try {
WikimediaLanguageCodes.getLanguageCode(lang);
return WikimediaLanguageCodes.fixLanguageCodeIfDeprecated(lang);
} catch(IllegalArgumentException e) {
} catch (IllegalArgumentException e) {
return null;
}
}
@Override
public String evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public String evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return _langId;
}
/**
* @return the language code for this language
*/
@ -58,7 +79,7 @@ public class WbLanguageConstant implements WbExpression<String> {
public String getLang() {
return _langId;
}
/**
* @return the name of the language in itself
*/
@ -66,19 +87,19 @@ public class WbLanguageConstant implements WbExpression<String> {
public String getLabel() {
return _langLabel;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbLanguageConstant.class.isInstance(other)) {
if (other == null || !WbLanguageConstant.class.isInstance(other)) {
return false;
}
WbLanguageConstant otherConstant = (WbLanguageConstant)other;
WbLanguageConstant otherConstant = (WbLanguageConstant) other;
return _langId.equals(otherConstant.getLang()) && _langLabel.equals(otherConstant.getLabel());
}
@Override
public int hashCode() {
return _langId.hashCode();
}
}

41
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -7,22 +30,22 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A language variable generates a language code from a cell.
* It checks its values against a known list of valid language codes
* and fixes on the fly the deprecated ones (see {@link WbLanguageConstant}).
* A language variable generates a language code from a cell. It checks its
* values against a known list of valid language codes and fixes on the fly the
* deprecated ones (see {@link WbLanguageConstant}).
*/
public class WbLanguageVariable extends WbVariableExpr<String> {
@JsonCreator
public WbLanguageVariable() {
public WbLanguageVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
* the name of the column the expression should draw its value from
*/
public WbLanguageVariable(String columnName) {
setColumnName(columnName);
@ -40,7 +63,7 @@ public class WbLanguageVariable extends WbVariableExpr<String> {
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLanguageVariable.class);

76
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -11,53 +34,52 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant for a geographical location. The accepted format is lat,lng or lat/lng.
* A constant for a geographical location. The accepted format is lat,lng or
* lat/lng.
*
* @author Antonin Delpeuch
*
*/
public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
public static final double defaultPrecision = GlobeCoordinatesValue.PREC_TEN_MICRO_DEGREE;
private String value;
private GlobeCoordinatesValue parsed;
@JsonCreator
public WbLocationConstant(
@JsonProperty("value") String origValue) throws ParseException {
public WbLocationConstant(@JsonProperty("value") String origValue) throws ParseException {
this.value = origValue;
Validate.notNull(origValue);
this.parsed = parse(origValue);
Validate.notNull(this.parsed);
}
/**
* Parses a string to a location.
*
* @param expr
* the string to parse
* @return
* the parsed location
* the string to parse
* @return the parsed location
* @throws ParseException
*/
public static GlobeCoordinatesValue parse(String expr) throws ParseException {
public static GlobeCoordinatesValue parse(String expr)
throws ParseException {
double lat = 0;
double lng = 0;
double precision = defaultPrecision;
String[] parts = expr.split("[,/]");
if (parts.length >= 2 && parts.length <= 3) {
try {
lat = Double.parseDouble(parts[0]);
lng = Double.parseDouble(parts[1]);
if (parts.length == 3) {
precision = Double.parseDouble(parts[2]);
}
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision,
GlobeCoordinatesValue.GLOBE_EARTH);
} catch(NumberFormatException e) {
;
}
try {
lat = Double.parseDouble(parts[0]);
lng = Double.parseDouble(parts[1]);
if (parts.length == 3) {
precision = Double.parseDouble(parts[2]);
}
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision, GlobeCoordinatesValue.GLOBE_EARTH);
} catch (NumberFormatException e) {
;
}
}
throw new ParseException("Invalid globe coordinates", 0);
}
@ -67,7 +89,7 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
throws SkipSchemaExpressionException {
return parsed;
}
/**
* @return the original value as a string.
*/
@ -75,16 +97,16 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
public String getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbLocationConstant.class.isInstance(other)) {
if (other == null || !WbLocationConstant.class.isInstance(other)) {
return false;
}
WbLocationConstant otherConstant = (WbLocationConstant)other;
WbLocationConstant otherConstant = (WbLocationConstant) other;
return value.equals(otherConstant.getValue());
}
@Override
public int hashCode() {
return value.hashCode();

30
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -9,12 +32,11 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
@JsonCreator
public WbLocationVariable() {
}
public WbLocationVariable(String columnName) {
@ -31,7 +53,7 @@ public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLocationVariable.class);

52
extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -10,15 +33,13 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
private WbExpression<? extends String> languageExpr;
private WbExpression<? extends StringValue> valueExpr;
@JsonCreator
public WbMonolingualExpr(
@JsonProperty("language") WbExpression<? extends String> languageExpr,
public WbMonolingualExpr(@JsonProperty("language") WbExpression<? extends String> languageExpr,
@JsonProperty("value") WbExpression<? extends StringValue> valueExpr) {
Validate.notNull(languageExpr);
this.languageExpr = languageExpr;
@ -33,13 +54,9 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
try {
String lang = getLanguageExpr().evaluate(ctxt);
return Datamodel.makeMonolingualTextValue(text, lang);
} catch(SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"monolingual-text-without-language",
null,
QAWarning.Severity.WARNING,
1);
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("monolingual-text-without-language", null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_text", text);
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
@ -55,17 +72,16 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
public WbExpression<? extends StringValue> getValueExpr() {
return valueExpr;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbMonolingualExpr.class.isInstance(other)) {
if (other == null || !WbMonolingualExpr.class.isInstance(other)) {
return false;
}
WbMonolingualExpr otherExpr = (WbMonolingualExpr)other;
return languageExpr.equals(otherExpr.getLanguageExpr()) &&
valueExpr.equals(otherExpr.getValueExpr());
WbMonolingualExpr otherExpr = (WbMonolingualExpr) other;
return languageExpr.equals(otherExpr.getLanguageExpr()) && valueExpr.equals(otherExpr.getValueExpr());
}
@Override
public int hashCode() {
return languageExpr.hashCode() + valueExpr.hashCode();

77
extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -10,56 +33,53 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* An expression that represent a term (label, description or alias).
* The structure is slightly different from other expressions because
* we need to call different methods on {@link ItemUpdateBuilder}.
* An expression that represent a term (label, description or alias). The
* structure is slightly different from other expressions because we need to
* call different methods on {@link ItemUpdateBuilder}.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbNameDescExpr {
enum NameDescrType {
LABEL,
DESCRIPTION,
ALIAS,
LABEL, DESCRIPTION, ALIAS,
}
private NameDescrType type;
private WbMonolingualExpr value;
@JsonCreator
public WbNameDescExpr(
@JsonProperty("name_type") NameDescrType type,
public WbNameDescExpr(@JsonProperty("name_type") NameDescrType type,
@JsonProperty("value") WbMonolingualExpr value) {
Validate.notNull(type);
this.type = type;
Validate.notNull(value);
this.value = value;
}
/**
* Evaluates the expression and adds the result to the item update.
*
* @param item
* the item update where the term should be stored
* the item update where the term should be stored
* @param ctxt
* the evaluation context for the expression
* the evaluation context for the expression
*/
public void contributeTo(ItemUpdateBuilder item, ExpressionContext ctxt) {
try {
MonolingualTextValue val = getValue().evaluate(ctxt);
switch (getType()) {
case LABEL:
item.addLabel(val);
break;
case DESCRIPTION:
item.addDescription(val);
break;
case ALIAS:
item.addAlias(val);
break;
case LABEL:
item.addLabel(val);
break;
case DESCRIPTION:
item.addDescription(val);
break;
case ALIAS:
item.addAlias(val);
break;
}
} catch (SkipSchemaExpressionException e) {
return;
@ -75,17 +95,16 @@ public class WbNameDescExpr {
public WbMonolingualExpr getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbNameDescExpr.class.isInstance(other)) {
if (other == null || !WbNameDescExpr.class.isInstance(other)) {
return false;
}
WbNameDescExpr otherExpr = (WbNameDescExpr)other;
return type.equals(otherExpr.getType()) &&
value.equals(otherExpr.getValue());
WbNameDescExpr otherExpr = (WbNameDescExpr) other;
return type.equals(otherExpr.getType()) && value.equals(otherExpr.getValue());
}
@Override
public int hashCode() {
return type.hashCode() + value.hashCode();

44
extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -14,15 +37,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
*
*/
public class WbPropConstant implements WbExpression<PropertyIdValue> {
private String pid;
private String label;
private String datatype;
@JsonCreator
public WbPropConstant(
@JsonProperty("pid") String pid,
@JsonProperty("label") String label,
public WbPropConstant(@JsonProperty("pid") String pid, @JsonProperty("label") String label,
@JsonProperty("datatype") String datatype) {
Validate.notNull(pid);
this.pid = pid;
@ -35,7 +56,7 @@ public class WbPropConstant implements WbExpression<PropertyIdValue> {
public PropertyIdValue evaluate(ExpressionContext ctxt) {
return new SuggestedPropertyIdValue(pid, ctxt.getBaseIRI(), label);
}
@JsonProperty("pid")
public String getPid() {
return pid;
@ -45,19 +66,20 @@ public class WbPropConstant implements WbExpression<PropertyIdValue> {
public String getLabel() {
return label;
}
@JsonProperty("datatype")
public String getDatatype() {
return datatype;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbPropConstant.class.isInstance(other)) {
if (other == null || !WbPropConstant.class.isInstance(other)) {
return false;
}
WbPropConstant otherConstant = (WbPropConstant)other;
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel()) && datatype.equals(otherConstant.getDatatype());
WbPropConstant otherConstant = (WbPropConstant) other;
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel())
&& datatype.equals(otherConstant.getDatatype());
}
@Override

52
extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.math.BigDecimal;
@ -13,22 +36,21 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbQuantityExpr implements WbExpression<QuantityValue> {
private final WbExpression<? extends StringValue> amountExpr;
private final WbExpression<? extends ItemIdValue> unitExpr;
/**
* Creates an expression for a quantity, which
* contains two sub-expressions: one for the amount (a string with
* a particular format) and one for the unit, which is optional.
* Creates an expression for a quantity, which contains two sub-expressions: one
* for the amount (a string with a particular format) and one for the unit,
* which is optional.
*
* Setting unitExpr to null will give quantities without units. Setting
* it to a non-null value will make the unit mandatory: if the unit expression
* fails to evaluate, the whole quantity expression will fail too.
* Setting unitExpr to null will give quantities without units. Setting it to a
* non-null value will make the unit mandatory: if the unit expression fails to
* evaluate, the whole quantity expression will fail too.
*/
@JsonCreator
public WbQuantityExpr(
@JsonProperty("amount") WbExpression<? extends StringValue> amountExpr,
public WbQuantityExpr(@JsonProperty("amount") WbExpression<? extends StringValue> amountExpr,
@JsonProperty("unit") WbExpression<? extends ItemIdValue> unitExpr) {
Validate.notNull(amountExpr);
this.amountExpr = amountExpr;
@ -40,22 +62,22 @@ public class WbQuantityExpr implements WbExpression<QuantityValue> {
throws SkipSchemaExpressionException {
StringValue amount = getLanguageExpr().evaluate(ctxt);
// we know the amount is nonnull, nonempty here
BigDecimal parsedAmount = null;
try {
parsedAmount = new BigDecimal(amount.getString());
} catch(NumberFormatException e) {
} catch (NumberFormatException e) {
throw new SkipSchemaExpressionException();
}
if(getUnitExpr() != null) {
if (getUnitExpr() != null) {
ItemIdValue unit = getUnitExpr().evaluate(ctxt);
return Datamodel.makeQuantityValue(parsedAmount, unit.getIri());
}
return Datamodel.makeQuantityValue(parsedAmount);
}
@JsonProperty("amount")
public WbExpression<? extends StringValue> getLanguageExpr() {
return amountExpr;

44
extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -22,19 +45,20 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use=JsonTypeInfo.Id.NONE)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbReferenceExpr implements WbExpression<Reference> {
private List<WbSnakExpr> snakExprs;
@JsonCreator
public WbReferenceExpr(
@JsonProperty("snaks") List<WbSnakExpr> snakExprs) {
public WbReferenceExpr(@JsonProperty("snaks") List<WbSnakExpr> snakExprs) {
Validate.notNull(snakExprs);
this.snakExprs = snakExprs;
}
@Override
public Reference evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public Reference evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (WbSnakExpr expr : getSnaks()) {
List<Snak> snakList = new ArrayList<Snak>(1);
@ -45,7 +69,7 @@ public class WbReferenceExpr implements WbExpression<Reference> {
continue;
}
}
if (! snakGroups.isEmpty()) {
if (!snakGroups.isEmpty()) {
return Datamodel.makeReference(snakGroups);
} else {
throw new SkipSchemaExpressionException();
@ -59,13 +83,13 @@ public class WbReferenceExpr implements WbExpression<Reference> {
@Override
public boolean equals(Object other) {
if(other == null || !WbReferenceExpr.class.isInstance(other)) {
if (other == null || !WbReferenceExpr.class.isInstance(other)) {
return false;
}
WbReferenceExpr otherExpr = (WbReferenceExpr)other;
WbReferenceExpr otherExpr = (WbReferenceExpr) other;
return snakExprs.equals(otherExpr.getSnaks());
}
@Override
public int hashCode() {
return snakExprs.hashCode();

39
extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -19,15 +42,14 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use=JsonTypeInfo.Id.NONE)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbSnakExpr implements WbExpression<Snak> {
private WbExpression<? extends PropertyIdValue> prop;
private WbExpression<? extends Value> value;
@JsonCreator
public WbSnakExpr(
@JsonProperty("prop") WbExpression<? extends PropertyIdValue> propExpr,
public WbSnakExpr(@JsonProperty("prop") WbExpression<? extends PropertyIdValue> propExpr,
@JsonProperty("value") WbExpression<? extends Value> valueExpr) {
Validate.notNull(propExpr);
this.prop = propExpr;
@ -36,7 +58,8 @@ public class WbSnakExpr implements WbExpression<Snak> {
}
@Override
public Snak evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public Snak evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = getProp().evaluate(ctxt);
Value evaluatedValue = value.evaluate(ctxt);
return Datamodel.makeValueSnak(propertyId, evaluatedValue);
@ -51,7 +74,7 @@ public class WbSnakExpr implements WbExpression<Snak> {
public WbExpression<? extends Value> getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbSnakExpr.class.isInstance(other)) {
@ -60,7 +83,7 @@ public class WbSnakExpr implements WbExpression<Snak> {
WbSnakExpr otherExpr = (WbSnakExpr) other;
return prop.equals(otherExpr.getProp()) && value.equals(otherExpr.getValue());
}
@Override
public int hashCode() {
return prop.hashCode() + value.hashCode();

69
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -10,10 +33,10 @@ import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.Value;
@ -24,14 +47,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbStatementExpr {
private WbExpression<? extends Value> mainSnakValueExpr;
private List<WbSnakExpr> qualifierExprs;
private List<WbReferenceExpr> referenceExprs;
@JsonCreator
public WbStatementExpr(
@JsonProperty("value") WbExpression<? extends Value> mainSnakValueExpr,
public WbStatementExpr(@JsonProperty("value") WbExpression<? extends Value> mainSnakValueExpr,
@JsonProperty("qualifiers") List<WbSnakExpr> qualifierExprs,
@JsonProperty("references") List<WbReferenceExpr> referenceExprs) {
Validate.notNull(mainSnakValueExpr);
@ -45,7 +67,7 @@ public class WbStatementExpr {
}
this.referenceExprs = referenceExprs;
}
public static List<SnakGroup> groupSnaks(List<Snak> snaks) {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (Snak snak : snaks) {
@ -55,23 +77,19 @@ public class WbStatementExpr {
}
return snakGroups;
}
public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId)
throws SkipSchemaExpressionException {
Value mainSnakValue = getMainsnak().evaluate(ctxt);
Snak mainSnak = Datamodel.makeValueSnak(propertyId, mainSnakValue);
// evaluate qualifiers
List<Snak> qualifiers = new ArrayList<Snak>(getQualifiers().size());
for (WbSnakExpr qExpr : getQualifiers()) {
try {
qualifiers.add(qExpr.evaluate(ctxt));
} catch(SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"ignored-qualifiers",
null,
QAWarning.Severity.INFO,
1);
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("ignored-qualifiers", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
@ -79,24 +97,20 @@ public class WbStatementExpr {
}
List<SnakGroup> groupedQualifiers = groupSnaks(qualifiers);
Claim claim = Datamodel.makeClaim(subject, mainSnak, groupedQualifiers);
// evaluate references
List<Reference> references = new ArrayList<Reference>();
for (WbReferenceExpr rExpr : getReferences()) {
try {
references.add(rExpr.evaluate(ctxt));
} catch(SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"ignored-references",
null,
QAWarning.Severity.INFO,
1);
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("ignored-references", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
}
}
StatementRank rank = StatementRank.NORMAL;
return Datamodel.makeStatement(claim, references, rank, "");
}
@ -115,18 +129,17 @@ public class WbStatementExpr {
public List<WbReferenceExpr> getReferences() {
return referenceExprs;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbStatementExpr.class.isInstance(other)) {
return false;
}
WbStatementExpr otherExpr = (WbStatementExpr)other;
return mainSnakValueExpr.equals(otherExpr.getMainsnak()) &&
qualifierExprs.equals(otherExpr.getQualifiers()) &&
referenceExprs.equals(otherExpr.getReferences());
WbStatementExpr otherExpr = (WbStatementExpr) other;
return mainSnakValueExpr.equals(otherExpr.getMainsnak()) && qualifierExprs.equals(otherExpr.getQualifiers())
&& referenceExprs.equals(otherExpr.getReferences());
}
@Override
public int hashCode() {
return mainSnakValueExpr.hashCode() + qualifierExprs.hashCode() + referenceExprs.hashCode();

45
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -14,17 +37,15 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbStatementGroupExpr {
private WbExpression<? extends PropertyIdValue> propertyExpr;
private List<WbStatementExpr> statementExprs;
@JsonCreator
public WbStatementGroupExpr(
@JsonProperty("property") WbExpression<? extends PropertyIdValue> propertyExpr,
public WbStatementGroupExpr(@JsonProperty("property") WbExpression<? extends PropertyIdValue> propertyExpr,
@JsonProperty("statements") List<WbStatementExpr> claimExprs) {
Validate.notNull(propertyExpr);
this.propertyExpr = propertyExpr;
@ -33,10 +54,11 @@ public class WbStatementGroupExpr {
this.statementExprs = claimExprs;
}
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) throws SkipSchemaExpressionException {
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = propertyExpr.evaluate(ctxt);
List<Statement> statements = new ArrayList<Statement>(statementExprs.size());
for(WbStatementExpr expr : statementExprs) {
for (WbStatementExpr expr : statementExprs) {
try {
statements.add(expr.evaluate(ctxt, subject, propertyId));
} catch (SkipSchemaExpressionException e) {
@ -59,17 +81,16 @@ public class WbStatementGroupExpr {
public List<WbStatementExpr> getStatements() {
return statementExprs;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbStatementGroupExpr.class.isInstance(other)) {
return false;
}
WbStatementGroupExpr otherExpr = (WbStatementGroupExpr)other;
return propertyExpr.equals(otherExpr.getProperty()) &&
statementExprs.equals(otherExpr.getStatements());
WbStatementGroupExpr otherExpr = (WbStatementGroupExpr) other;
return propertyExpr.equals(otherExpr.getProperty()) && statementExprs.equals(otherExpr.getStatements());
}
@Override
public int hashCode() {
return propertyExpr.hashCode() + statementExprs.hashCode();

32
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -7,22 +30,21 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbStringConstant implements WbExpression<StringValue> {
private String value;
@JsonCreator
public WbStringConstant(@JsonProperty("value") String value) {
Validate.notNull(value);
this.value = value;
}
@Override
public StringValue evaluate(ExpressionContext ctxt) {
return Datamodel.makeStringValue(value);
}
@JsonProperty("value")
public String getValue() {
return value;

35
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -15,17 +38,17 @@ import com.google.refine.model.Cell;
*
*/
public class WbStringVariable extends WbVariableExpr<StringValue> {
@JsonCreator
public WbStringVariable() {
public WbStringVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
* the name of the column the expression should draw its value from
*/
public WbStringVariable(String columnName) {
setColumnName(columnName);
@ -39,7 +62,7 @@ public class WbStringVariable extends WbVariableExpr<StringValue> {
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbStringVariable.class);

79
extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -8,18 +31,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
/**
* A base class for expressions which draw their values
* from a particular column.
* A base class for expressions which draw their values from a particular
* column.
*
* @author antonin
* @author Antonin Delpeuch
*
* @param <T>
* the type of Wikibase value returned by the expression.
* the type of Wikibase value returned by the expression.
*/
public abstract class WbVariableExpr<T> implements WbExpression<T> {
private String columnName;
/**
* Constructs a variable without setting the column name yet.
*/
@ -27,21 +50,20 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
public WbVariableExpr() {
columnName = null;
}
/**
* Returns the column name used by the variable.
* @return
* the OpenRefine column name
*
* @return the OpenRefine column name
*/
@JsonProperty("columnName")
public String getColumnName() {
return columnName;
}
/**
* Changes the column name used by the variable.
* This is useful for deserialization, as well as updates when
* column names change.
* Changes the column name used by the variable. This is useful for
* deserialization, as well as updates when column names change.
*/
@JsonProperty("columnName")
public void setColumnName(String columnName) {
@ -60,43 +82,42 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
}
throw new SkipSchemaExpressionException();
}
/**
* Method that should be implemented by subclasses,
* converting an OpenRefine cell to a Wikibase value.
* Access to other values and emiting warnings is possible via
* the supplied EvaluationContext object.
* Method that should be implemented by subclasses, converting an OpenRefine
* cell to a Wikibase value. Access to other values and emiting warnings is
* possible via the supplied EvaluationContext object.
*
* @param cell
* the cell to convert
* the cell to convert
* @param ctxt
* the evaluation context
* @return
* the corresponding Wikibase value
* the evaluation context
* @return the corresponding Wikibase value
*/
public abstract T fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException;
public abstract T fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException;
/**
* Helper for equality methods of subclasses.
*
* @param other
* the object to compare
* the object to compare
* @param columnName
* the column name to compare to
* the column name to compare to
* @param targetClass
* the target class for equality
* the target class for equality
* @return
*/
protected boolean equalAsVariables(Object other, Class<? extends WbVariableExpr<?>> targetClass) {
if(other == null || !targetClass.isInstance(other)) {
if (other == null || !targetClass.isInstance(other)) {
return false;
}
return columnName.equals(targetClass.cast(other).getColumnName());
}
@Override
public int hashCode() {
return columnName.hashCode();
}
}

129
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -8,6 +31,10 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -17,17 +44,10 @@ import com.google.refine.browsing.RowVisitor;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.openrefine.wikidata.schema.WbItemDocumentExpr;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.ExpressionContext;
import org.openrefine.wikidata.utils.JacksonJsonizable;
/**
* Main class representing a skeleton of Wikibase edits with
* OpenRefine columns as variables.
* Main class representing a skeleton of Wikibase edits with OpenRefine columns
* as variables.
*
* @author Antonin Delpeuch
*
@ -35,18 +55,18 @@ import org.openrefine.wikidata.utils.JacksonJsonizable;
public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
protected List<WbItemDocumentExpr> itemDocumentExprs = new ArrayList<WbItemDocumentExpr>();
protected String baseIri = "http://www.wikidata.org/entity/";
/**
* Constructor.
*/
*/
public WikibaseSchema() {
}
/**
* @return the site IRI of the Wikibase instance referenced by this schema
*/
@ -60,24 +80,24 @@ public class WikibaseSchema implements OverlayModel {
public List<WbItemDocumentExpr> getItemDocumentExpressions() {
return itemDocumentExprs;
}
public void setItemDocumentExpressions(List<WbItemDocumentExpr> exprs) {
this.itemDocumentExprs = exprs;
}
/**
* Evaluates all item documents in a particular expression context.
* This specifies, among others, a row where the values of the variables
* will be read.
* Evaluates all item documents in a particular expression context. This
* specifies, among others, a row where the values of the variables will be
* read.
*
* @param ctxt
* the context in which the schema should be evaluated.
* the context in which the schema should be evaluated.
* @return
*/
public List<ItemUpdate> evaluateItemDocuments(ExpressionContext ctxt) {
List<ItemUpdate> result = new ArrayList<>();
for (WbItemDocumentExpr expr : itemDocumentExprs) {
try {
result.add(expr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
@ -86,24 +106,23 @@ public class WikibaseSchema implements OverlayModel {
}
return result;
}
/**
* Evaluates the schema on a project, returning a list of ItemUpdates
* generated by the schema.
* Evaluates the schema on a project, returning a list of ItemUpdates generated
* by the schema.
*
* Some warnings will be emitted in the warning store: those are only
* the ones that are generated at evaluation time (such as invalid formats
* for dates). Issues detected on candidate statements (such as constraint
* violations) are not included at this stage.
* Some warnings will be emitted in the warning store: those are only the ones
* that are generated at evaluation time (such as invalid formats for dates).
* Issues detected on candidate statements (such as constraint violations) are
* not included at this stage.
*
* @param project
* the project on which the schema should be evaluated
* the project on which the schema should be evaluated
* @param engine
* the engine, which gives access to the current facets
* the engine, which gives access to the current facets
* @param warningStore
* a store in which issues will be emitted
* @return item updates are stored in their
* generating order (not merged yet).
* a store in which issues will be emitted
* @return item updates are stored in their generating order (not merged yet).
*/
public List<ItemUpdate> evaluate(Project project, Engine engine, QAWarningStore warningStore) {
List<ItemUpdate> result = new ArrayList<>();
@ -111,35 +130,32 @@ public class WikibaseSchema implements OverlayModel {
filteredRows.accept(project, new EvaluatingRowVisitor(result, warningStore));
return result;
}
/**
* Same as above, ignoring any warnings.
*/
public List<ItemUpdate> evaluate(Project project, Engine engine) {
return evaluate(project, engine, null);
}
protected class EvaluatingRowVisitor implements RowVisitor {
private List<ItemUpdate> result;
private QAWarningStore warningStore;
public EvaluatingRowVisitor(List<ItemUpdate> result, QAWarningStore warningStore) {
this.result = result;
this.warningStore = warningStore;
}
@Override
public void start(Project project) {
;
;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
ExpressionContext ctxt = new ExpressionContext(
baseIri,
rowIndex,
row,
project.columnModel,
warningStore);
ExpressionContext ctxt = new ExpressionContext(baseIri, rowIndex, row, project.columnModel, warningStore);
result.addAll(evaluateItemDocuments(ctxt));
return false;
}
@ -150,12 +166,14 @@ public class WikibaseSchema implements OverlayModel {
}
}
static public WikibaseSchema reconstruct(JSONObject o) throws JSONException {
static public WikibaseSchema reconstruct(JSONObject o)
throws JSONException {
JSONArray changeArr = o.getJSONArray("itemDocuments");
WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) {
WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i), WbItemDocumentExpr.class);
WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i),
WbItemDocumentExpr.class);
schema.itemDocumentExprs.add(changeExpr);
}
return schema;
@ -173,30 +191,31 @@ public class WikibaseSchema implements OverlayModel {
writer.endArray();
writer.endObject();
}
static public WikibaseSchema load(Project project, JSONObject obj) throws Exception {
static public WikibaseSchema load(Project project, JSONObject obj)
throws Exception {
return reconstruct(obj);
}
@Override
public void onBeforeSave(Project project) {
}
@Override
public void onAfterSave(Project project) {
}
@Override
public void dispose(Project project) {
}
@Override
public boolean equals(Object other) {
if(other == null || !WikibaseSchema.class.isInstance(other)) {
if (other == null || !WikibaseSchema.class.isInstance(other)) {
return false;
}
WikibaseSchema otherSchema = (WikibaseSchema)other;
WikibaseSchema otherSchema = (WikibaseSchema) other;
return itemDocumentExprs.equals(otherSchema.getItemDocumentExpressions());
}
}
}

54
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/PrefetchedEntityIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.List;
@ -5,34 +28,31 @@ import java.util.List;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
/**
* An entity id value that also comes with
* a label and possibly types.
* An entity id value that also comes with a label and possibly types.
*
* The rationale behind this classes is that OpenRefine
* already stores labels and types for the Wikidata items
* it knows about (in the reconciliation data), so it is
* worth keeping this data to avoid re-fetching it when
* we need it.
* The rationale behind this classes is that OpenRefine already stores labels
* and types for the Wikidata items it knows about (in the reconciliation data),
* so it is worth keeping this data to avoid re-fetching it when we need it.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public interface PrefetchedEntityIdValue extends EntityIdValue {
/**
* This should return the label "as we got it", with no guarantee
* that it is current or that its language matches that of the user.
* In general though, that should be the case if the user always uses
* OpenRefine with the same language settings.
* This should return the label "as we got it", with no guarantee that it is
* current or that its language matches that of the user. In general though,
* that should be the case if the user always uses OpenRefine with the same
* language settings.
*
* @return the preferred label of the entity
*/
public String getLabel();
/**
* Returns a list of types for this item. Again these are the types
* as they were originally fetched from the reconciliation interface:
* they can diverge from what is currently on the item.
* Returns a list of types for this item. Again these are the types as they were
* originally fetched from the reconciliation interface: they can diverge from
* what is currently on the item.
*
* Empty lists should be returned for
*/

85
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.ArrayList;
@ -14,43 +37,41 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.refine.model.Recon;
/**
* An EntityIdValue that holds not just the id but also
* the label as fetched by either the reconciliation interface
* or the suggester and its type, both stored as reconciliation
* candidates.
* An EntityIdValue that holds not just the id but also the label as fetched by
* either the reconciliation interface or the suggester and its type, both
* stored as reconciliation candidates.
*
* This label will be localized depending on the language chosen
* by the user for OpenRefine's interface. Storing it lets us
* reuse it later on without having to re-fetch it.
* This label will be localized depending on the language chosen by the user for
* OpenRefine's interface. Storing it lets us reuse it later on without having
* to re-fetch it.
*
* Storing the types also lets us perform some constraint checks
* without re-fetching the types of many items.
* Storing the types also lets us perform some constraint checks without
* re-fetching the types of many items.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
private Recon _recon;
private String _cellValue;
public ReconEntityIdValue(Recon match, String cellValue) {
_recon = match;
_cellValue = cellValue;
assert (Recon.Judgment.Matched.equals(_recon.judgment) ||
Recon.Judgment.New.equals(_recon.judgment));
assert (Recon.Judgment.Matched.equals(_recon.judgment) || Recon.Judgment.New.equals(_recon.judgment));
}
@JsonIgnore
public boolean isMatched() {
return Recon.Judgment.Matched.equals(_recon.judgment) && _recon.match != null;
}
@JsonIgnore
public boolean isNew() {
return !isMatched();
}
public String getLabel() {
if (isMatched()) {
return _recon.match.name;
@ -66,32 +87,28 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
return new ArrayList<>();
}
}
@Override
public abstract String getEntityType();
/**
* Returns the integer used internally in OpenRefine to identify the new
* item.
* Returns the integer used internally in OpenRefine to identify the new item.
*
* @return
* the judgment history entry id of the reconciled cell
* @return the judgment history entry id of the reconciled cell
*/
public long getReconInternalId() {
return getRecon().judgmentHistoryEntry;
}
/**
* Returns the reconciliation object corresponding to this entity.
*
* @return
* the full reconciliation metadata of the corresponding cell
* @return the full reconciliation metadata of the corresponding cell
*/
public Recon getRecon() {
return _recon;
}
/**
* Returns the id of the reconciled item
*/
@ -100,9 +117,9 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
if (isMatched()) {
return _recon.match.id;
} else if (ET_ITEM.equals(getEntityType())) {
return "Q"+getReconInternalId();
return "Q" + getReconInternalId();
} else if (ET_PROPERTY.equals(getEntityType())) {
return "P"+getReconInternalId();
return "P" + getReconInternalId();
}
return null;
}
@ -125,13 +142,13 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
public <T> T accept(ValueVisitor<T> valueVisitor) {
return valueVisitor.visit(this);
}
@Override
public boolean equals(Object other) {
return Equality.equalsEntityIdValue(this, other);
}
@Override
public int hashCode() {
return Hash.hashCode(this);
@ -139,10 +156,10 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
@Override
public String toString() {
if(isNew()) {
return "new item (reconciled from " + getReconInternalId() +")";
if (isNew()) {
return "new item (reconciled from " + getReconInternalId() + ")";
} else {
return getIri() + " (reconciled from " + getReconInternalId()+")";
return getIri() + " (reconciled from " + getReconInternalId() + ")";
}
}
}

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconItemIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconPropertyIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;

49
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java
View File

@ -1,22 +1,44 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.ArrayList;
import java.util.List;
import org.wikidata.wdtk.datamodel.helpers.Hash;
import org.wikidata.wdtk.datamodel.helpers.ToString;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* An EntityIdValue that we have obtained from a suggest widget
* in the schema alignment dialog.
* An EntityIdValue that we have obtained from a suggest widget in the schema
* alignment dialog.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue {
private String _id;
private String _siteIRI;
private String _label;
@ -31,22 +53,22 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
public String getId() {
return _id;
}
@Override
public String getSiteIri() {
return _siteIRI;
}
@Override
public String getLabel() {
return _label;
}
@Override
public List<String> getTypes() {
return new ArrayList<>();
}
@Override
public String getIri() {
return getSiteIri() + getId();
@ -56,14 +78,13 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
public <T> T accept(ValueVisitor<T> valueVisitor) {
return valueVisitor.visit(this);
}
@Override
public boolean equals(Object other) {
if (other == null ||
!EntityIdValue.class.isInstance(other)) {
return false;
if (other == null || !EntityIdValue.class.isInstance(other)) {
return false;
}
final EntityIdValue otherNew = (EntityIdValue)other;
final EntityIdValue otherNew = (EntityIdValue) other;
return getIri().equals(otherNew.getIri());
}

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedItemIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;

25
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.helpers.ToString;
@ -16,6 +39,6 @@ public class SuggestedPropertyIdValue extends SuggestedEntityIdValue implements
@Override
public String toString() {
return "suggested "+ToString.toString(this)+" (\""+getLabel()+"\")";
return "suggested " + ToString.toString(this) + " (\"" + getLabel() + "\")";
}
}

25
extensions/wikidata/src/org/openrefine/wikidata/schema/exceptions/InvalidSchemaException.java
View File

@ -1,6 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.exceptions;
public class InvalidSchemaException extends Exception {
static final long serialVersionUID = 494837587034L;
}

25
extensions/wikidata/src/org/openrefine/wikidata/schema/exceptions/SkipSchemaExpressionException.java
View File

@ -1,6 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.exceptions;
public class SkipSchemaExpressionException extends Exception {
static final long serialVersionUID = 738592057L;
}

166
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates;
import java.util.ArrayList;
@ -23,42 +46,43 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to plan an update of an item, after evaluating the statements
* but before fetching the current content of the item (this is why it does not
* A class to plan an update of an item, after evaluating the statements but
* before fetching the current content of the item (this is why it does not
* extend StatementsUpdate).
*
* @author Antonin Delpeuch
*/
public class ItemUpdate {
private final ItemIdValue qid;
private final List<Statement> addedStatements;
private final Set<Statement> deletedStatements;
private final Set<MonolingualTextValue> labels;
private final Set<MonolingualTextValue> descriptions;
private final Set<MonolingualTextValue> aliases;
/**
* Constructor.
*
* @param qid
* the subject of the document. It can be a reconciled item value for new items.
* the subject of the document. It can be a reconciled item value for
* new items.
* @param addedStatements
* the statements to add on the item. They should be distinct. They
* are modelled as a list because their insertion order matters.
* the statements to add on the item. They should be distinct. They
* are modelled as a list because their insertion order matters.
* @param deletedStatements
* the statements to remove from the item
* the statements to remove from the item
* @param labels
* the labels to add on the item
* the labels to add on the item
* @param descriptions
* the descriptions to add on the item
* the descriptions to add on the item
* @param aliases
* the aliases to add on the item. In theory their order should matter
* but in practice people rarely rely on the order of aliases so this
* is just kept as a set for simplicity.
* the aliases to add on the item. In theory their order should
* matter but in practice people rarely rely on the order of aliases
* so this is just kept as a set for simplicity.
*/
@JsonCreator
public ItemUpdate(
@JsonProperty("subject") ItemIdValue qid,
public ItemUpdate(@JsonProperty("subject") ItemIdValue qid,
@JsonProperty("addedStatements") List<Statement> addedStatements,
@JsonProperty("deletedStatements") Set<Statement> deletedStatements,
@JsonProperty("labels") Set<MonolingualTextValue> labels,
@ -66,28 +90,28 @@ public class ItemUpdate {
@JsonProperty("addedAliases") Set<MonolingualTextValue> aliases) {
Validate.notNull(qid);
this.qid = qid;
if(addedStatements == null) {
if (addedStatements == null) {
addedStatements = Collections.emptyList();
}
this.addedStatements = addedStatements;
if(deletedStatements == null) {
if (deletedStatements == null) {
deletedStatements = Collections.emptySet();
}
this.deletedStatements = deletedStatements;
if(labels == null) {
if (labels == null) {
labels = Collections.emptySet();
}
this.labels = labels;
if(descriptions == null) {
if (descriptions == null) {
descriptions = Collections.emptySet();
}
this.descriptions = descriptions;
if(aliases == null) {
if (aliases == null) {
aliases = Collections.emptySet();
}
this.aliases = aliases;
}
/**
* @return the subject of the item
*/
@ -95,10 +119,10 @@ public class ItemUpdate {
public ItemIdValue getItemId() {
return qid;
}
/**
* Added statements are recorded as a list because
* their order of insertion matters.
* Added statements are recorded as a list because their order of insertion
* matters.
*
* @return the list of all added statements
*/
@ -106,7 +130,7 @@ public class ItemUpdate {
public List<Statement> getAddedStatements() {
return addedStatements;
}
/**
* @return the list of all deleted statements
*/
@ -114,7 +138,7 @@ public class ItemUpdate {
public Set<Statement> getDeletedStatements() {
return deletedStatements;
}
/**
* @return the list of updated labels
*/
@ -122,7 +146,7 @@ public class ItemUpdate {
public Set<MonolingualTextValue> getLabels() {
return labels;
}
/**
* @return the list of updated descriptions
*/
@ -130,7 +154,7 @@ public class ItemUpdate {
public Set<MonolingualTextValue> getDescriptions() {
return descriptions;
}
/**
* @return the list of updated aliases
*/
@ -138,7 +162,7 @@ public class ItemUpdate {
public Set<MonolingualTextValue> getAliases() {
return aliases;
}
/**
* @return true when this change is empty and its subject is not new
*/
@ -146,30 +170,27 @@ public class ItemUpdate {
public boolean isNull() {
return isEmpty() && !isNew();
}
/**
* @return true when this change leaves the content of the document untouched
*/
@JsonIgnore
public boolean isEmpty() {
return (addedStatements.isEmpty()
&& deletedStatements.isEmpty()
&& labels.isEmpty()
&& descriptions.isEmpty()
return (addedStatements.isEmpty() && deletedStatements.isEmpty() && labels.isEmpty() && descriptions.isEmpty()
&& aliases.isEmpty());
}
/**
* Merges all the changes in other into this instance.
* Both updates should have the same subject.
* Merges all the changes in other into this instance. Both updates should have
* the same subject.
*
* @param other
* the other change that should be merged
* the other change that should be merged
*/
public ItemUpdate merge(ItemUpdate other) {
Validate.isTrue(qid.equals(other.getItemId()));
List<Statement> newAddedStatements = new ArrayList<>(addedStatements);
for(Statement statement : other.getAddedStatements()) {
for (Statement statement : other.getAddedStatements()) {
if (!newAddedStatements.contains(statement)) {
newAddedStatements.add(statement);
}
@ -182,20 +203,17 @@ public class ItemUpdate {
newDescriptions.addAll(other.getDescriptions());
Set<MonolingualTextValue> newAliases = new HashSet<>(aliases);
newAliases.addAll(other.getAliases());
return new ItemUpdate(
qid, newAddedStatements, newDeletedStatements,
newLabels, newDescriptions, newAliases);
return new ItemUpdate(qid, newAddedStatements, newDeletedStatements, newLabels, newDescriptions, newAliases);
}
/**
* Group added statements in StatementGroups: useful if the
* item is new.
* Group added statements in StatementGroups: useful if the item is new.
*
* @return a grouped version of getAddedStatements()
*/
public List<StatementGroup> getAddedStatementGroups() {
Map<PropertyIdValue, List<Statement>> map = new HashMap<>();
for(Statement statement : getAddedStatements()) {
for (Statement statement : getAddedStatements()) {
PropertyIdValue propertyId = statement.getClaim().getMainSnak().getPropertyId();
if (!map.containsKey(propertyId)) {
map.put(propertyId, new ArrayList<Statement>());
@ -203,26 +221,26 @@ public class ItemUpdate {
map.get(propertyId).add(statement);
}
List<StatementGroup> result = new ArrayList<>();
for(Map.Entry<PropertyIdValue, List<Statement>> entry : map.entrySet()) {
for (Map.Entry<PropertyIdValue, List<Statement>> entry : map.entrySet()) {
result.add(new StatementGroupImpl(entry.getValue()));
}
return result;
}
/**
* Group a list of ItemUpdates by subject: this is useful to make one single edit
* per item.
* Group a list of ItemUpdates by subject: this is useful to make one single
* edit per item.
*
* @param itemDocuments
* @return a map from item ids to merged ItemUpdate for that id
*/
public static Map<EntityIdValue, ItemUpdate> groupBySubject(List<ItemUpdate> itemDocuments) {
Map<EntityIdValue, ItemUpdate> map = new HashMap<>();
for(ItemUpdate update : itemDocuments) {
for (ItemUpdate update : itemDocuments) {
if (update.isNull()) {
continue;
}
ItemIdValue qid = update.getItemId();
if (map.containsKey(qid)) {
ItemUpdate oldUpdate = map.get(qid);
@ -233,59 +251,53 @@ public class ItemUpdate {
}
return map;
}
/**
* Is this update about a new item?
*/
public boolean isNew() {
return EntityIdValue.SITE_LOCAL.equals(getItemId().getSiteIri());
}
/**
* This should only be used when creating a new item.
* This ensures that we never add an alias without adding
* a label in the same language.
* This should only be used when creating a new item. This ensures that we never
* add an alias without adding a label in the same language.
*/
public ItemUpdate normalizeLabelsAndAliases() {
// Ensure that we are only adding aliases with labels
Set<String> labelLanguages = labels.stream()
.map(l -> l.getLanguageCode())
.collect(Collectors.toSet());
Set<String> labelLanguages = labels.stream().map(l -> l.getLanguageCode()).collect(Collectors.toSet());
Set<MonolingualTextValue> filteredAliases = new HashSet<>();
Set<MonolingualTextValue> newLabels = new HashSet<>(labels);
for(MonolingualTextValue alias : aliases) {
if(!labelLanguages.contains(alias.getLanguageCode())) {
for (MonolingualTextValue alias : aliases) {
if (!labelLanguages.contains(alias.getLanguageCode())) {
labelLanguages.add(alias.getLanguageCode());
newLabels.add(alias);
} else {
filteredAliases.add(alias);
}
}
return new ItemUpdate(qid, addedStatements, deletedStatements,
newLabels, descriptions, filteredAliases);
return new ItemUpdate(qid, addedStatements, deletedStatements, newLabels, descriptions, filteredAliases);
}
@Override
public boolean equals(Object other) {
if(other == null || !ItemUpdate.class.isInstance(other)) {
if (other == null || !ItemUpdate.class.isInstance(other)) {
return false;
}
ItemUpdate otherUpdate = (ItemUpdate)other;
return qid.equals(otherUpdate.getItemId())&&
addedStatements.equals(otherUpdate.getAddedStatements()) &&
deletedStatements.equals(otherUpdate.getDeletedStatements()) &&
labels.equals(otherUpdate.getLabels()) &&
descriptions.equals(otherUpdate.getDescriptions()) &&
aliases.equals(otherUpdate.getAliases());
ItemUpdate otherUpdate = (ItemUpdate) other;
return qid.equals(otherUpdate.getItemId()) && addedStatements.equals(otherUpdate.getAddedStatements())
&& deletedStatements.equals(otherUpdate.getDeletedStatements())
&& labels.equals(otherUpdate.getLabels()) && descriptions.equals(otherUpdate.getDescriptions())
&& aliases.equals(otherUpdate.getAliases());
}
@Override
public int hashCode() {
return qid.hashCode() + addedStatements.hashCode() + deletedStatements.hashCode() +
labels.hashCode() + descriptions.hashCode() + aliases.hashCode();
return qid.hashCode() + addedStatements.hashCode() + deletedStatements.hashCode() + labels.hashCode()
+ descriptions.hashCode() + aliases.hashCode();
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
@ -317,5 +329,5 @@ public class ItemUpdate {
builder.append("\n>");
return builder.toString();
}
}

104
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates;
import java.util.Set;
@ -10,7 +33,6 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* Constructs a {@link ItemUpdate} incrementally.
*
@ -18,6 +40,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
*
*/
public class ItemUpdateBuilder {
private ItemIdValue qid;
private List<Statement> addedStatements;
private Set<Statement> deletedStatements;
@ -25,12 +48,13 @@ public class ItemUpdateBuilder {
private Set<MonolingualTextValue> descriptions;
private Set<MonolingualTextValue> aliases;
private boolean built;
/**
* Constructor.
*
* @param qid
* the subject of the document. It can be a reconciled item value for new items.
* the subject of the document. It can be a reconciled item value for
* new items.
*/
public ItemUpdateBuilder(ItemIdValue qid) {
Validate.notNull(qid);
@ -42,50 +66,50 @@ public class ItemUpdateBuilder {
this.aliases = new HashSet<MonolingualTextValue>();
this.built = false;
}
/**
* Mark a statement for insertion. If it matches an existing
* statement, it will update the statement instead.
* Mark a statement for insertion. If it matches an existing statement, it will
* update the statement instead.
*
* @param statement
* the statement to add or update
* the statement to add or update
*/
public ItemUpdateBuilder addStatement(Statement statement) {
Validate.isTrue(!built, "ItemUpdate has already been built");
addedStatements.add(statement);
return this;
}
/**
* Mark a statement for deletion. If no such statement exists,
* nothing will be deleted.
* Mark a statement for deletion. If no such statement exists, nothing will be
* deleted.
*
* @param statement
* the statement to delete
* the statement to delete
*/
public ItemUpdateBuilder deleteStatement(Statement statement) {
Validate.isTrue(!built, "ItemUpdate has already been built");
deletedStatements.add(statement);
return this;
}
/**
* Add a list of statement, as in {@link addStatement}.
*
* @param statements
* the statements to add
* the statements to add
*/
public ItemUpdateBuilder addStatements(Set<Statement> statements) {
Validate.isTrue(!built, "ItemUpdate has already been built");
addedStatements.addAll(statements);
return this;
}
/**
* Delete a list of statements, as in {@link deleteStatement}.
*
* @param statements
* the statements to delete
* the statements to delete
*/
public ItemUpdateBuilder deleteStatements(Set<Statement> statements) {
Validate.isTrue(!built, "ItemUpdate has already been built");
@ -94,24 +118,24 @@ public class ItemUpdateBuilder {
}
/**
* Adds a label to the item. It will override any
* existing label in this language.
* Adds a label to the item. It will override any existing label in this
* language.
*
* @param label
* the label to add
* the label to add
*/
public ItemUpdateBuilder addLabel(MonolingualTextValue label) {
Validate.isTrue(!built, "ItemUpdate has already been built");
labels.add(label);
return this;
}
/**
* Adds a list of labels to the item. It will override any
* existing label in each language.
* Adds a list of labels to the item. It will override any existing label in
* each language.
*
* @param labels
* the labels to add
* the labels to add
*/
public ItemUpdateBuilder addLabels(Set<MonolingualTextValue> labels) {
Validate.isTrue(!built, "ItemUpdate has already been built");
@ -120,24 +144,24 @@ public class ItemUpdateBuilder {
}
/**
* Adds a description to the item. It will override any existing
* description in this language.
* Adds a description to the item. It will override any existing description in
* this language.
*
* @param description
* the description to add
* the description to add
*/
public ItemUpdateBuilder addDescription(MonolingualTextValue description) {
Validate.isTrue(!built, "ItemUpdate has already been built");
descriptions.add(description);
return this;
}
/**
* Adds a list of descriptions to the item. It will override any
* existing description in each language.
* Adds a list of descriptions to the item. It will override any existing
* description in each language.
*
* @param descriptions
* the descriptions to add
* the descriptions to add
*/
public ItemUpdateBuilder addDescriptions(Set<MonolingualTextValue> descriptions) {
Validate.isTrue(!built, "ItemUpdate has already been built");
@ -146,39 +170,39 @@ public class ItemUpdateBuilder {
}
/**
* Adds an alias to the item. It will be added to any existing
* aliases in that language.
* Adds an alias to the item. It will be added to any existing aliases in that
* language.
*
* @param alias
* the alias to add
* the alias to add
*/
public ItemUpdateBuilder addAlias(MonolingualTextValue alias) {
Validate.isTrue(!built, "ItemUpdate has already been built");
aliases.add(alias);
aliases.add(alias);
return this;
}
/**
* Adds a list of aliases to the item. They will be added to any
* existing aliases in each language.
* Adds a list of aliases to the item. They will be added to any existing
* aliases in each language.
*
* @param aliases
* the aliases to add
* the aliases to add
*/
public ItemUpdateBuilder addAliases(Set<MonolingualTextValue> aliases) {
Validate.isTrue(!built, "ItemUpdate has already been built");
this.aliases.addAll(aliases);
return this;
}
/**
* Constructs the {@link ItemUpdate}.
*
* @return
*/
public ItemUpdate build() {
built = true;
return new ItemUpdate(qid, addedStatements, deletedStatements,
labels, descriptions, aliases);
return new ItemUpdate(qid, addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

24
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/ImpossibleSchedulingException.java
View File

@ -1,6 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
public class ImpossibleSchedulingException extends Exception {
private static final long serialVersionUID = 6621563898380564148L;

76
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.Collections;
@ -5,7 +28,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -21,33 +43,29 @@ import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* A class that extracts the new entity ids referred to
* in a statement.
* A class that extracts the new entity ids referred to in a statement.
*
* @author Antonin Delpeuch
*
*/
public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
/**
* Extracts all the new entities mentioned by this statement. This
* does not include the subject of the statement.
* Extracts all the new entities mentioned by this statement. This does not
* include the subject of the statement.
*
* @param statement
* the statement to inspect
* @return
* the set of all new entities mentioned by the statement
* the statement to inspect
* @return the set of all new entities mentioned by the statement
*/
public Set<ReconItemIdValue> extractPointers(Statement statement) {
Set<ReconItemIdValue> result = new HashSet<>();
result.addAll(extractPointers(statement.getClaim().getMainSnak()));
result.addAll(extractPointers(statement.getClaim().getQualifiers()));
statement.getReferences().stream()
.map(l -> extractPointers(l.getSnakGroups()))
.forEach(s -> result.addAll(s));
statement.getReferences().stream().map(l -> extractPointers(l.getSnakGroups())).forEach(s -> result.addAll(s));
return result;
}
/**
* Extracts all the new entities mentioned by this list of snak groups.
*
@ -56,12 +74,10 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
*/
public Set<ReconItemIdValue> extractPointers(List<SnakGroup> snakGroups) {
Set<ReconItemIdValue> result = new HashSet<>();
snakGroups.stream()
.map(s -> extractPointers(s))
.forEach(s -> result.addAll(s));
return result;
snakGroups.stream().map(s -> extractPointers(s)).forEach(s -> result.addAll(s));
return result;
}
/***
* Extracts all the new entities mentioned by this snak group.
*
@ -70,16 +86,14 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
*/
public Set<ReconItemIdValue> extractPointers(SnakGroup snakGroup) {
Set<ReconItemIdValue> result = new HashSet<>();
snakGroup.getSnaks().stream()
.map(s -> extractPointers(s))
.forEach(s -> result.addAll(s));
return result;
snakGroup.getSnaks().stream().map(s -> extractPointers(s)).forEach(s -> result.addAll(s));
return result;
}
/**
* Extracts all new entities mentioned by this snak group.
* Currently there will be at most one: the target of the snak
* (as property ids cannot be new for now).
* Extracts all new entities mentioned by this snak group. Currently there will
* be at most one: the target of the snak (as property ids cannot be new for
* now).
*
* @param snak
* @return
@ -90,7 +104,7 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
result.addAll(extractPointers(snak.getValue()));
return result;
}
/**
* Extracts any new entity from the value.
*
@ -115,9 +129,9 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
@Override
public Set<ReconItemIdValue> visit(EntityIdValue value) {
if(ReconItemIdValue.class.isInstance(value)) {
ReconItemIdValue recon = (ReconItemIdValue)value;
if(recon.isNew()) {
if (ReconItemIdValue.class.isInstance(value)) {
ReconItemIdValue recon = (ReconItemIdValue) value;
if (recon.isNew()) {
return Collections.singleton(recon);
}
}
@ -146,7 +160,7 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
}
@Override
public Set<ReconItemIdValue> visit(TimeValue value) {
public Set<ReconItemIdValue> visit(TimeValue value) {
return null;
}
}

84
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/QuickStatementsUpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -8,49 +31,45 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class QuickStatementsUpdateScheduler implements UpdateScheduler {
private PointerExtractor extractor = new PointerExtractor();
/**
* This map holds for each new entity id value a list of updates
* that refer to this id (and should hence be scheduled right after
* creation of that entity).
* This map holds for each new entity id value a list of updates that refer to
* this id (and should hence be scheduled right after creation of that entity).
*/
private Map<ItemIdValue, UpdateSequence> pointerUpdates;
/**
* This contains all updates which do not refer to any new entity
* apart from possibly the subject, in the order that they were supplied to us.
* This contains all updates which do not refer to any new entity apart from
* possibly the subject, in the order that they were supplied to us.
*/
private UpdateSequence pointerFreeUpdates;
/**
* Separates out the statements which refer to new items from the rest
* of the update. The resulting updates are stored in {@link referencingUpdates}
* and {@link updatesWithoutReferences}.
* Separates out the statements which refer to new items from the rest of the
* update. The resulting updates are stored in {@link referencingUpdates} and
* {@link updatesWithoutReferences}.
*
* @param update
* @throws ImpossibleSchedulingException
* if two new item ids are referred to in the same statement
* @throws ImpossibleSchedulingException
* if two new item ids are referred to in the same statement
*/
protected void splitUpdate(ItemUpdate update) throws ImpossibleSchedulingException {
protected void splitUpdate(ItemUpdate update)
throws ImpossibleSchedulingException {
ItemUpdateBuilder remainingUpdateBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels())
.addDescriptions(update.getDescriptions())
.addAliases(update.getAliases())
.addLabels(update.getLabels()).addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
Map<ItemIdValue, ItemUpdateBuilder> referencingUpdates = new HashMap<>();
for(Statement statement : update.getAddedStatements()) {
for (Statement statement : update.getAddedStatements()) {
Set<ReconItemIdValue> pointers = extractor.extractPointers(statement);
if (pointers.isEmpty()) {
remainingUpdateBuilder.addStatement(statement);
@ -68,14 +87,14 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
throw new ImpossibleSchedulingException();
}
}
// Add the update that is not referring to anything to the schedule
ItemUpdate pointerFree = remainingUpdateBuilder.build();
if (!pointerFree.isNull()) {
pointerFreeUpdates.add(pointerFree);
}
// Add the other updates to the map
for(Entry<ItemIdValue, ItemUpdateBuilder> entry : referencingUpdates.entrySet()) {
for (Entry<ItemIdValue, ItemUpdateBuilder> entry : referencingUpdates.entrySet()) {
ItemUpdate pointerUpdate = entry.getValue().build();
UpdateSequence pointerUpdatesForKey = pointerUpdates.get(entry.getKey());
if (pointerUpdatesForKey == null) {
@ -87,18 +106,19 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
}
@Override
public List<ItemUpdate> schedule(List<ItemUpdate> updates) throws ImpossibleSchedulingException {
public List<ItemUpdate> schedule(List<ItemUpdate> updates)
throws ImpossibleSchedulingException {
pointerUpdates = new HashMap<>();
pointerFreeUpdates = new UpdateSequence();
for(ItemUpdate update : updates) {
for (ItemUpdate update : updates) {
splitUpdate(update);
}
// Reconstruct
List<ItemUpdate> fullSchedule = new ArrayList<>();
Set<ItemIdValue> mentionedNewEntities = new HashSet<>(pointerUpdates.keySet());
for(ItemUpdate update : pointerFreeUpdates.getUpdates()) {
Set<ItemIdValue> mentionedNewEntities = new HashSet<>(pointerUpdates.keySet());
for (ItemUpdate update : pointerFreeUpdates.getUpdates()) {
fullSchedule.add(update);
UpdateSequence backPointers = pointerUpdates.get(update.getItemId());
if (backPointers != null) {
@ -106,11 +126,11 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
}
mentionedNewEntities.remove(update.getItemId());
}
// Create any item that was referred to but untouched
// (this is just for the sake of correctness, it would be bad to do that
// as the items would remain blank in this batch).
for(ItemIdValue missingId : mentionedNewEntities) {
for (ItemIdValue missingId : mentionedNewEntities) {
fullSchedule.add(new ItemUpdateBuilder(missingId).build());
fullSchedule.addAll(pointerUpdates.get(missingId).getUpdates());
}

51
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/UpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.List;
@ -5,28 +28,26 @@ import java.util.List;
import org.openrefine.wikidata.updates.ItemUpdate;
/**
* A scheduling strategy for item updates.
* Given a list of initial updates, the scheduler
* reorganizes these updates (possibly splitting them
* or merging them) to create a sequence that is suitable
* for a particular import process.
* A scheduling strategy for item updates. Given a list of initial updates, the
* scheduler reorganizes these updates (possibly splitting them or merging them)
* to create a sequence that is suitable for a particular import process.
*
* @author Antonin Delpeuch
*
*/
public interface UpdateScheduler {
/**
* Performs the scheduling. The initial updates are provided
* as a list so that the scheduler can attempt to respect the
* initial order (but no guarantee is made for that in general).
* Performs the scheduling. The initial updates are provided as a list so that
* the scheduler can attempt to respect the initial order (but no guarantee is
* made for that in general).
*
* @param updates
* the updates to schedule
* @return
* the reorganized updates
* @throws ImpossibleSchedulingException
* when the scheduler cannot cope with a particular edit plan.
* the updates to schedule
* @return the reorganized updates
* @throws ImpossibleSchedulingException
* when the scheduler cannot cope with a particular edit plan.
*/
public List<ItemUpdate> schedule(List<ItemUpdate> updates) throws ImpossibleSchedulingException;
public List<ItemUpdate> schedule(List<ItemUpdate> updates)
throws ImpossibleSchedulingException;
}

42
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/UpdateSequence.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -10,12 +33,13 @@ import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
/**
* Helper class to store a list of updates where each subject
* appears at most once. It preserves order of insertion.
* Helper class to store a list of updates where each subject appears at most
* once. It preserves order of insertion.
*
* @author Antonin Delpeuch
*/
public class UpdateSequence {
/**
* The list of updates stored by this container
*/
@ -24,16 +48,16 @@ public class UpdateSequence {
* An index to keep track of where each item is touched in the sequence
*/
private Map<ItemIdValue, Integer> index = new HashMap<>();
/**
* Adds a new update to the list, merging it with any existing
* one with the same subject.
* Adds a new update to the list, merging it with any existing one with the same
* subject.
*
* @param update
*/
public void add(ItemUpdate update) {
ItemIdValue subject = update.getItemId();
if(index.containsKey(subject)) {
if (index.containsKey(subject)) {
int i = index.get(subject);
ItemUpdate oldUpdate = updates.get(i);
updates.set(i, oldUpdate.merge(update));
@ -42,18 +66,18 @@ public class UpdateSequence {
updates.add(update);
}
}
/**
* @return the list of merged updates
*/
public List<ItemUpdate> getUpdates() {
return updates;
}
/**
* @return the set of touched subjects
*/
public Set<ItemIdValue> getSubjects() {
return index.keySet();
}
}
}

77
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/WikibaseAPIUpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -15,33 +38,32 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A simple scheduler for batches commited via the Wikibase API.
*
* The strategy is quite simple and makes at most two edits
* per touched item (which is not minimal though). Each update
* is split between statements making references to new items,
* and statements not making these references. All updates with no
* references to new items are done first (which creates all new
* The strategy is quite simple and makes at most two edits per touched item
* (which is not minimal though). Each update is split between statements making
* references to new items, and statements not making these references. All
* updates with no references to new items are done first (which creates all new
* items), then all other updates are done.
*
* @author Antonin Delpeuch
*
*/
public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
/**
* The first part of updates: the ones which create new items
* without referring to any other new item.
* The first part of updates: the ones which create new items without referring
* to any other new item.
*/
private UpdateSequence pointerFreeUpdates;
/**
* The second part of the updates: all existing items, plus
* all parts of new items that refer to other new items.
* The second part of the updates: all existing items, plus all parts of new
* items that refer to other new items.
*/
private UpdateSequence pointerFullUpdates;
/**
* The set of all new items referred to in the whole batch.
*/
private Set<ItemIdValue> allPointers;
private PointerExtractor extractor = new PointerExtractor();
@Override
@ -50,22 +72,20 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
pointerFreeUpdates = new UpdateSequence();
pointerFullUpdates = new UpdateSequence();
allPointers = new HashSet<>();
for(ItemUpdate update : updates) {
for (ItemUpdate update : updates) {
splitUpdate(update);
}
// Part 1: add all the pointer free updates
result.addAll(pointerFreeUpdates.getUpdates());
// Part 1': add the remaining new items that have not been touched
Set<ItemIdValue> unseenPointers = new HashSet<>(allPointers);
unseenPointers.removeAll(pointerFreeUpdates.getSubjects());
result.addAll(unseenPointers.stream()
.map(e -> new ItemUpdateBuilder(e).build())
.collect(Collectors.toList()));
result.addAll(unseenPointers.stream().map(e -> new ItemUpdateBuilder(e).build()).collect(Collectors.toList()));
// Part 2: add all the pointer full updates
result.addAll(pointerFullUpdates.getUpdates());
@ -74,17 +94,16 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
/**
* Splits an update into two parts
*
* @param update
*/
protected void splitUpdate(ItemUpdate update) {
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels())
.addDescriptions(update.getDescriptions())
.addAliases(update.getAliases())
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId()).addLabels(update.getLabels())
.addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
ItemUpdateBuilder pointerFullBuilder = new ItemUpdateBuilder(update.getItemId());
for(Statement statement : update.getAddedStatements()) {
for (Statement statement : update.getAddedStatements()) {
Set<ReconItemIdValue> pointers = extractor.extractPointers(statement);
if (pointers.isEmpty()) {
pointerFreeBuilder.addStatement(statement);
@ -93,8 +112,8 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
}
allPointers.addAll(pointers);
}
if(update.isNew()) {
if (update.isNew()) {
// If the update is new, we might need to split it
// in two (if it refers to any other new entity).
ItemUpdate pointerFree = pointerFreeBuilder.build();
@ -111,5 +130,5 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
pointerFullUpdates.add(update);
}
}
}

61
extensions/wikidata/src/org/openrefine/wikidata/utils/EntityCache.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.util.concurrent.TimeUnit;
@ -14,42 +37,42 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
public class EntityCache {
private static EntityCache _entityCache = new EntityCache();
private LoadingCache<String, EntityDocument> _cache = null;
private WikibaseDataFetcher _fetcher;
private EntityCache() {
ApiConnection connection = ApiConnection.getWikidataApiConnection();
_fetcher = new WikibaseDataFetcher(connection, Datamodel.SITE_WIKIDATA);
_cache = CacheBuilder.newBuilder()
.maximumSize(4096)
.expireAfterWrite(1, TimeUnit.HOURS)
.build(
new CacheLoader<String, EntityDocument>() {
public EntityDocument load(String entityId) throws Exception {
EntityDocument doc = _fetcher.getEntityDocument(entityId);
if (doc != null) {
return doc;
} else {
throw new MediaWikiApiErrorException("400", "Unknown entity id \""+entityId+"\"");
}
}
});
_cache = CacheBuilder.newBuilder().maximumSize(4096).expireAfterWrite(1, TimeUnit.HOURS)
.build(new CacheLoader<String, EntityDocument>() {
public EntityDocument load(String entityId)
throws Exception {
EntityDocument doc = _fetcher.getEntityDocument(entityId);
if (doc != null) {
return doc;
} else {
throw new MediaWikiApiErrorException("400", "Unknown entity id \"" + entityId + "\"");
}
}
});
}
public EntityDocument get(EntityIdValue id) {
return _cache.apply(id.getId());
}
public static EntityCache getEntityCache() {
if (_entityCache == null) {
_entityCache = new EntityCache();
}
return _entityCache;
}
public static EntityDocument getEntityDocument(EntityIdValue id) {
return getEntityCache().get(id);
}

43
extensions/wikidata/src/org/openrefine/wikidata/utils/FirstLinesExtractor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.io.IOException;
@ -6,25 +29,27 @@ import java.io.StringReader;
import java.io.StringWriter;
public class FirstLinesExtractor {
/**
* Returns the first n lines of a given string
*
* @param content
* the content, where lines are separated by '\n'
* the content, where lines are separated by '\n'
* @param nbLines
* the number of lines to extract
* @return
* the first lines of the string
* @throws IOException
* the number of lines to extract
* @return the first lines of the string
* @throws IOException
*/
public static String extractFirstLines(String content, int nbLines) throws IOException {
public static String extractFirstLines(String content, int nbLines)
throws IOException {
StringWriter stringWriter = new StringWriter();
LineNumberReader reader = new LineNumberReader(new StringReader(content));
// Only keep the first 50 lines
reader.setLineNumber(0);
String line = reader.readLine();
for(int i = 1; i != nbLines && line != null; i++) {
stringWriter.write(line+"\n");
for (int i = 1; i != nbLines && line != null; i++) {
stringWriter.write(line + "\n");
line = reader.readLine();
}
if (reader.getLineNumber() == nbLines) {

41
extensions/wikidata/src/org/openrefine/wikidata/utils/JacksonJsonizable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.io.IOException;
@ -16,13 +39,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.Jsonizable;
/**
* This class is inefficient because it serializes the
* object to string and then deserializes it back. Unfortunately,
* this is the only simple way to bridge Jackson to org.json.
* This conversion should be removed when (if ?) we migrate OpenRefine
* a better JSON library.
* This class is inefficient because it serializes the object to string and then
* deserializes it back. Unfortunately, this is the only simple way to bridge
* Jackson to org.json. This conversion should be removed when (if ?) we migrate
* OpenRefine a better JSON library.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@ -38,8 +60,9 @@ public abstract class JacksonJsonizable implements Jsonizable {
throw new JSONException(e.toString());
}
}
public static <T> T fromJSONClass(JSONObject obj, Class<T> klass) throws JSONException {
public static <T> T fromJSONClass(JSONObject obj, Class<T> klass)
throws JSONException {
ObjectMapper mapper = new ObjectMapper();
String json = obj.toString();
try {
@ -50,7 +73,7 @@ public abstract class JacksonJsonizable implements Jsonizable {
throw new JSONException(e.toString());
} catch (IOException e) {
throw new JSONException(e.toString());
}
}
}
}

62
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java
View File

@ -1,57 +1,73 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
public abstract class CommandTest extends RefineTest {
protected Project project = null;
protected HttpServletRequest request = null;
protected HttpServletResponse response = null;
protected StringWriter writer = null;
protected Command command = null;
@BeforeMethod(alwaysRun = true)
public void setUpProject() throws JSONException {
public void setUpProject()
throws JSONException {
project = createCSVProject(TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
request = mock(HttpServletRequest.class);
response = mock(HttpServletResponse.class);
writer = new StringWriter();
PrintWriter printWriter = new PrintWriter(writer);
when(request.getParameter("project")).thenReturn(String.valueOf(project.id));
try {
when(response.getWriter()).thenReturn(printWriter);
} catch (IOException e1) {
Assert.fail();
}
}
}

50
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java
View File

@ -1,5 +1,34 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
import java.io.IOException;
import javax.servlet.ServletException;
import org.json.JSONException;
@ -8,29 +37,24 @@ import org.openrefine.wikidata.testing.TestingData;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
import java.io.IOException;
import com.google.refine.util.ParsingUtilities;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest {
@BeforeMethod
public void SetUp() throws JSONException {
public void SetUp()
throws JSONException {
command = new PreviewWikibaseSchemaCommand();
}
@Test
public void testValidSchema() throws JSONException, IOException, ServletException {
public void testValidSchema()
throws JSONException, IOException, ServletException {
String schemaJson = jsonFromFile("data/schema/inception.json").toString();
when(request.getParameter("schema")).thenReturn(schemaJson);
command.doPost(request, response);
JSONObject response = ParsingUtilities.evaluateJsonStringToObject(writer.toString());
assertEquals(TestingData.inceptionWithNewQS, response.getString("quickstatements"));
}

43
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommandTest.java
View File

@ -1,31 +1,54 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
import java.io.IOException;
import javax.servlet.ServletException;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class SaveWikibaseSchemaCommandTest extends SchemaCommandTest {
@BeforeMethod
public void setUp() {
this.command = new SaveWikibaseSchemaCommand();
}
@Test
public void testValidSchema() throws ServletException, IOException {
public void testValidSchema()
throws ServletException, IOException {
String schemaJson = jsonFromFile("data/schema/inception.json").toString();
when(request.getParameter("schema")).thenReturn(schemaJson);
command.doPost(request, response);
assertTrue(writer.toString().contains("\"ok\""));
}
}

42
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java
View File

@ -1,8 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
@ -10,20 +30,24 @@ import java.io.IOException;
import javax.servlet.ServletException;
import org.testng.annotations.Test;
public abstract class SchemaCommandTest extends CommandTest {
@Test
public void testNoSchema() throws ServletException, IOException {
public void testNoSchema()
throws ServletException, IOException {
command.doPost(request, response);
assertEquals("{\"status\":\"error\",\"message\":\"No Wikibase schema provided.\"}", writer.toString());
}
@Test
public void testInvalidSchema() throws ServletException, IOException {
public void testInvalidSchema()
throws ServletException, IOException {
when(request.getParameter("schema")).thenReturn("{bogus json");
command.doPost(request, response);
assertEquals("{\"status\":\"error\",\"message\":\"Wikibase schema could not be parsed.\"}", writer.toString());
}
}

96
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import org.openrefine.wikidata.testing.TestingData;
@ -31,14 +54,13 @@ import java.util.stream.Collectors;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
public class EditBatchProcessorTest extends RefineTest {
private WikibaseDataFetcher fetcher = null;
private WikibaseDataEditor editor = null;
private NewItemLibrary library = null;
private String summary = "my fantastic edits";
@BeforeMethod
public void setUp() {
fetcher = mock(WikibaseDataFetcher.class);
@ -46,33 +68,29 @@ public class EditBatchProcessorTest extends RefineTest {
editor.disableEditing(); // just in case we got mocking wrong…
library = new NewItemLibrary();
}
@Test
public void testNewItem() throws InterruptedException, MediaWikiApiErrorException, IOException {
public void testNewItem()
throws InterruptedException, MediaWikiApiErrorException, IOException {
List<ItemUpdate> batch = new ArrayList<>();
batch.add(new ItemUpdateBuilder(TestingData.existingId)
.addAlias(Datamodel.makeMonolingualTextValue("my new alias", "en"))
.addStatement(TestingData.generateStatement(TestingData.existingId, TestingData.newIdA))
.build());
.addStatement(TestingData.generateStatement(TestingData.existingId, TestingData.newIdA)).build());
MonolingualTextValue label = Datamodel.makeMonolingualTextValue("better label", "en");
batch.add(new ItemUpdateBuilder(TestingData.newIdA)
.addAlias(label)
.build());
batch.add(new ItemUpdateBuilder(TestingData.newIdA).addAlias(label).build());
// Plan expected edits
ItemDocument existingItem = ItemDocumentBuilder.forItemId(TestingData.existingId)
.withLabel(Datamodel.makeMonolingualTextValue("pomme", "fr"))
.withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr"))
.build();
.withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr")).build();
when(fetcher.getEntityDocuments(Collections.singletonList(TestingData.existingId.getId())))
.thenReturn(Collections.singletonMap(TestingData.existingId.getId(), existingItem));
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA)
.withLabel(label).build();
.thenReturn(Collections.singletonMap(TestingData.existingId.getId(), existingItem));
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234"))
.withLabel(label).withRevisionId(37828L).build();
when(editor.createItemDocument(expectedNewItem, summary)).thenReturn(createdNewItem);
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, 50);
assertEquals(2, processor.remainingEdits());
assertEquals(0, processor.progress());
@ -85,67 +103,63 @@ public class EditBatchProcessorTest extends RefineTest {
processor.performEdit(); // does not do anything
assertEquals(0, processor.remainingEdits());
assertEquals(100, processor.progress());
NewItemLibrary expectedLibrary = new NewItemLibrary();
expectedLibrary.setQid(1234L, "Q1234");
assertEquals(expectedLibrary, library);
}
@Test
public void testMultipleBatches() throws MediaWikiApiErrorException, InterruptedException, IOException {
public void testMultipleBatches()
throws MediaWikiApiErrorException, InterruptedException, IOException {
// Prepare test data
MonolingualTextValue description = Datamodel.makeMonolingualTextValue("village in Nepal", "en");
List<String> ids = new ArrayList<>();
for(int i = 124; i < 190; i++) {
ids.add("Q"+String.valueOf(i));
for (int i = 124; i < 190; i++) {
ids.add("Q" + String.valueOf(i));
}
List<ItemIdValue> qids = ids.stream()
.map(e -> Datamodel.makeWikidataItemIdValue(e))
List<ItemIdValue> qids = ids.stream().map(e -> Datamodel.makeWikidataItemIdValue(e))
.collect(Collectors.toList());
List<ItemUpdate> batch = qids.stream()
.map(qid -> new ItemUpdateBuilder(qid)
.addDescription(description)
.build())
.map(qid -> new ItemUpdateBuilder(qid).addDescription(description).build())
.collect(Collectors.toList());
int batchSize = 50;
List<ItemDocument> fullBatch = qids.stream()
.map(qid -> ItemDocumentBuilder.forItemId(qid)
.withStatement(TestingData.generateStatement(qid, TestingData.existingId))
.build())
.withStatement(TestingData.generateStatement(qid, TestingData.existingId)).build())
.collect(Collectors.toList());
List<ItemDocument> firstBatch = fullBatch.subList(0, batchSize);
List<ItemDocument> secondBatch = fullBatch.subList(batchSize, fullBatch.size());
when(fetcher.getEntityDocuments(toQids(firstBatch))).thenReturn(toMap(firstBatch));
when(fetcher.getEntityDocuments(toQids(secondBatch))).thenReturn(toMap(secondBatch));
// Run edits
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, batchSize);
assertEquals(0, processor.progress());
for(int i = 124; i < 190; i++) {
assertEquals(processor.remainingEdits(), 190-i);
for (int i = 124; i < 190; i++) {
assertEquals(processor.remainingEdits(), 190 - i);
processor.performEdit();
}
assertEquals(0, processor.remainingEdits());
assertEquals(100, processor.progress());
// Check result
assertEquals(new NewItemLibrary(), library);
verify(fetcher, times(1)).getEntityDocuments(toQids(firstBatch));
verify(fetcher, times(1)).getEntityDocuments(toQids(secondBatch));
for(ItemDocument doc : fullBatch) {
for (ItemDocument doc : fullBatch) {
verify(editor, times(1)).updateTermsStatements(doc, Collections.emptyList(),
Collections.singletonList(description), Collections.emptyList(), Collections.emptyList(),
Collections.emptyList(), Collections.emptyList(), summary);
}
}
private Map<String, EntityDocument> toMap(List<ItemDocument> docs) {
return docs.stream()
.collect(Collectors.toMap(doc -> doc.getItemId().getId(), doc -> doc));
return docs.stream().collect(Collectors.toMap(doc -> doc.getItemId().getId(), doc -> doc));
}
private List<String> toQids(List<ItemDocument> docs) {
return docs.stream().map(doc -> doc.getItemId().getId()).collect(Collectors.toList());
}

36
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import static org.junit.Assert.assertEquals;
@ -13,20 +36,21 @@ import com.google.refine.model.Recon;
import com.google.refine.tests.RefineTest;
public class NewItemLibraryTest extends RefineTest {
private NewItemLibrary library;
@BeforeMethod
public void setUp() {
library = new NewItemLibrary();
library.setQid(1234L, "Q345");
library.setQid(3289L, "Q384");
}
@Test
public void testRetrieveItem() {
assertEquals("Q345", library.getQid(1234L));
}
@Test
public void testUpdateReconciledCells() {
Project project = createCSVProject(TestingData.inceptionWithNewCsv);
@ -45,18 +69,18 @@ public class NewItemLibraryTest extends RefineTest {
isMatchedTo("Q865528", project.rows.get(1).cells.get(0));
isNewTo(1234L, project.rows.get(2).cells.get(0));
}
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(NewItemLibrary.class, library,
"{\"qidMap\":{\"1234\":\"Q345\",\"3289\":\"Q384\"}}");
}
private void isMatchedTo(String qid, Cell cell) {
assertEquals(Recon.Judgment.Matched, cell.recon.judgment);
assertEquals(qid, cell.recon.match.id);
}
private void isNewTo(long id, Cell cell) {
assertEquals(Recon.Judgment.New, cell.recon.judgment);
assertEquals(id, cell.recon.judgmentHistoryEntry);

45
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/ReconEntityRewriterTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import static org.junit.Assert.assertEquals;
@ -11,39 +34,39 @@ import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
public class ReconEntityRewriterTest {
NewItemLibrary library = null;
ReconEntityRewriter rewriter = null;
ItemIdValue subject = TestingData.newIdA;
ItemIdValue newlyCreated = Datamodel.makeWikidataItemIdValue("Q1234");
@BeforeMethod
public void setUp() {
library = new NewItemLibrary();
rewriter = new ReconEntityRewriter(library, subject);
}
@Test(expectedExceptions=IllegalArgumentException.class)
@Test(expectedExceptions = IllegalArgumentException.class)
public void testNotCreatedYet() {
rewriter.copy(TestingData.newIdB);
}
@Test
public void testSuccessfulRewrite() {
library.setQid(4567L, "Q1234");
assertEquals(newlyCreated, rewriter.copy(TestingData.newIdB));
}
@Test
public void testSubjectNotRewriten() {
assertEquals(subject, rewriter.copy(subject));
}
@Test
public void testMatched() {
assertEquals(TestingData.matchedId, rewriter.copy(TestingData.matchedId));
}
@Test
public void testRewriteUpdate() {
library.setQid(4567L, "Q1234");
@ -52,16 +75,14 @@ public class ReconEntityRewriterTest {
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de"))
.build();
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
ItemUpdate rewritten = rewriter.rewrite(update);
ItemUpdate expected = new ItemUpdateBuilder(subject)
.addStatement(TestingData.generateStatement(subject, newlyCreated))
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de"))
.build();
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
assertEquals(expected, rewritten);
}
}

93
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java
View File

@ -1,7 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.math.BigDecimal;
@ -14,100 +36,101 @@ import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
public class QSValuePrinterTest {
private QSValuePrinter printer;
public QSValuePrinterTest() {
printer = new QSValuePrinter();
}
void assertPrints(String expectedFormat, Value datavalue) {
assertEquals(expectedFormat, datavalue.accept(printer));
}
// Entity id values
@Test
public void printItemId() {
assertPrints("Q42", Datamodel.makeWikidataItemIdValue("Q42"));
}
@Test
public void printPropertyId() {
assertPrints("P42", Datamodel.makeWikidataPropertyIdValue("P42"));
}
@Test
public void printNewItemId() {
ReconEntityIdValue id = TestingData.makeNewItemIdValue(12345L, "my new item");
assertEquals("LAST", id.accept(printer));
// because no entity was previously created
ReconEntityIdValue differentId = TestingData.makeMatchedItemIdValue("Q78", "my existing item");
assertEquals("Q78", differentId.accept(printer));
}
// Globe coordinates
@Test
public void printGlobeCoordinate() {
// I don't see how to avoid the trailing zeros - in any case it's not a big deal because
// the precision is governed by a different parameter that QuickStatements does not support.
// I don't see how to avoid the trailing zeros - in any case it's not a big deal
// because
// the precision is governed by a different parameter that QuickStatements does
// not support.
assertPrints("@43.261930/10.927080", Datamodel.makeGlobeCoordinatesValue(43.26193, 10.92708,
GlobeCoordinatesValue.PREC_DEGREE, GlobeCoordinatesValue.GLOBE_EARTH));
}
// Monolingual text values
@Test
public void printMonolingualTextValue() {
assertPrints("pl:\"Krzyżacy\"", Datamodel.makeMonolingualTextValue("Krzyżacy", "pl"));
}
// Quantity values
@Test
public void printSimpleQuantityValue() {
assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
null, null, "1"));
assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null, "1"));
}
@Test
public void printQuantityValueWithUnit() {
assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
null, null, "http://www.wikidata.org/entity/Q11573"));
assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null,
"http://www.wikidata.org/entity/Q11573"));
}
@Test
public void printQuantityValueWithBounds() {
assertPrints("10.00[9.0,11.05]", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
new BigDecimal("9.0"), new BigDecimal("11.05"), "1"));
assertPrints("10.00[9.0,11.05]", Datamodel.makeQuantityValue(new BigDecimal("10.00"), new BigDecimal("9.0"),
new BigDecimal("11.05"), "1"));
}
@Test
public void printFullQuantity() {
assertPrints("10.00[9.0,11.05]U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
new BigDecimal("9.0"), new BigDecimal("11.05"), "http://www.wikidata.org/entity/Q11573"));
}
// String values
@Test
public void printString() {
assertPrints("\"hello\"", Datamodel.makeStringValue("hello"));
}
// Time values
@Test
public void printYear() {
assertPrints("+1586-00-00T00:00:00Z/9", Datamodel.makeTimeValue(1586L, (byte)0, (byte)0, (byte)0,
(byte)0, (byte)0, (byte)9, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
assertPrints("+1586-00-00T00:00:00Z/9", Datamodel.makeTimeValue(1586L, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte) 0, (byte) 9, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
}
@Test
public void printDay() {
assertPrints("+1586-03-09T00:00:00Z/11", Datamodel.makeTimeValue(1586L, (byte)3, (byte)9, (byte)0,
(byte)0, (byte)0, (byte)11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
assertPrints("+1586-03-09T00:00:00Z/11", Datamodel.makeTimeValue(1586L, (byte) 3, (byte) 9, (byte) 0, (byte) 0,
(byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
}
}

204
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import static org.junit.Assert.assertEquals;
@ -14,7 +37,6 @@ import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.updates.scheduler.UpdateSchedulerTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
@ -30,95 +52,93 @@ import com.google.refine.tests.RefineTest;
public class QuickStatementsExporterTest extends RefineTest {
private QuickStatementsExporter exporter = new QuickStatementsExporter();
private ItemIdValue newIdA = TestingData.makeNewItemIdValue(1234L, "new item A");
private ItemIdValue newIdB = TestingData.makeNewItemIdValue(5678L, "new item B");
private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377");
private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528");
private String export(ItemUpdate... itemUpdates) throws IOException {
StringWriter writer = new StringWriter();
exporter.translateItemList(Arrays.asList(itemUpdates), writer);
return writer.toString();
}
@Test
public void testSimpleProject() throws JSONException, IOException {
Project project = this.createCSVProject(
TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
JSONObject serialized = TestingData.jsonFromFile("data/schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
project.overlayModels.put("wikibaseSchema", schema);
Engine engine = new Engine(project);
StringWriter writer = new StringWriter();
Properties properties = new Properties();
exporter.export(project, properties, engine, writer);
assertEquals(TestingData.inceptionWithNewQS, writer.toString());
}
@Test
public void testImpossibleScheduling() throws IOException {
Statement sNewAtoNewB = TestingData.generateStatement(newIdA, newIdB);
ItemUpdate update = new ItemUpdateBuilder(newIdA).addStatement(sNewAtoNewB).build();
assertEquals(QuickStatementsExporter.impossibleSchedulingErrorMessage,
export(update));
}
@Test
public void testNameDesc() throws IOException {
ItemUpdate update = new ItemUpdateBuilder(newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("my new item", "en"))
.addDescription(Datamodel.makeMonolingualTextValue("isn't it awesome?", "en"))
.addAlias(Datamodel.makeMonolingualTextValue("fabitem", "en"))
.build();
assertEquals("CREATE\n"+
"LAST\tLen\t\"my new item\"\n"+
"LAST\tDen\t\"isn't it awesome?\"\n"+
"LAST\tAen\t\"fabitem\"\n",
export(update));
}
@Test
public void testDeleteStatement() throws IOException {
ItemUpdate update = new ItemUpdateBuilder(qid1)
.deleteStatement(TestingData.generateStatement(qid1, qid2))
.build();
assertEquals("- Q1377\tP38\tQ865528\n", export(update));
}
@Test
public void testQualifier() throws IOException {
Statement baseStatement = TestingData.generateStatement(qid1, qid2);
Statement otherStatement = TestingData.generateStatement(qid2, qid1);
Snak qualifierSnak = otherStatement.getClaim().getMainSnak();
SnakGroup group = Datamodel.makeSnakGroup(Collections.singletonList(qualifierSnak));
Claim claim = Datamodel.makeClaim(qid1, baseStatement.getClaim().getMainSnak(),
Collections.singletonList(group));
Statement statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
ItemUpdate update = new ItemUpdateBuilder(qid1)
.addStatement(statement)
.build();
assertEquals("Q1377\tP38\tQ865528\tP38\tQ1377\n", export(update));
}
@Test
public void testNoSchema() throws IOException {
Project project = this.createCSVProject("a,b\nc,d");
Engine engine = new Engine(project);
StringWriter writer = new StringWriter();
Properties properties = new Properties();
exporter.export(project, properties, engine, writer);
assertEquals(QuickStatementsExporter.noSchemaErrorMessage, writer.toString());
}
@Test
public void testGetContentType() {
assertEquals("text/plain", exporter.getContentType());
}
private QuickStatementsExporter exporter = new QuickStatementsExporter();
private ItemIdValue newIdA = TestingData.makeNewItemIdValue(1234L, "new item A");
private ItemIdValue newIdB = TestingData.makeNewItemIdValue(5678L, "new item B");
private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377");
private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528");
private String export(ItemUpdate... itemUpdates)
throws IOException {
StringWriter writer = new StringWriter();
exporter.translateItemList(Arrays.asList(itemUpdates), writer);
return writer.toString();
}
@Test
public void testSimpleProject()
throws JSONException, IOException {
Project project = this.createCSVProject(TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
JSONObject serialized = TestingData.jsonFromFile("data/schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
project.overlayModels.put("wikibaseSchema", schema);
Engine engine = new Engine(project);
StringWriter writer = new StringWriter();
Properties properties = new Properties();
exporter.export(project, properties, engine, writer);
assertEquals(TestingData.inceptionWithNewQS, writer.toString());
}
@Test
public void testImpossibleScheduling()
throws IOException {
Statement sNewAtoNewB = TestingData.generateStatement(newIdA, newIdB);
ItemUpdate update = new ItemUpdateBuilder(newIdA).addStatement(sNewAtoNewB).build();
assertEquals(QuickStatementsExporter.impossibleSchedulingErrorMessage, export(update));
}
@Test
public void testNameDesc()
throws IOException {
ItemUpdate update = new ItemUpdateBuilder(newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("my new item", "en"))
.addDescription(Datamodel.makeMonolingualTextValue("isn't it awesome?", "en"))
.addAlias(Datamodel.makeMonolingualTextValue("fabitem", "en")).build();
assertEquals("CREATE\n" + "LAST\tLen\t\"my new item\"\n" + "LAST\tDen\t\"isn't it awesome?\"\n"
+ "LAST\tAen\t\"fabitem\"\n", export(update));
}
@Test
public void testDeleteStatement()
throws IOException {
ItemUpdate update = new ItemUpdateBuilder(qid1).deleteStatement(TestingData.generateStatement(qid1, qid2))
.build();
assertEquals("- Q1377\tP38\tQ865528\n", export(update));
}
@Test
public void testQualifier()
throws IOException {
Statement baseStatement = TestingData.generateStatement(qid1, qid2);
Statement otherStatement = TestingData.generateStatement(qid2, qid1);
Snak qualifierSnak = otherStatement.getClaim().getMainSnak();
SnakGroup group = Datamodel.makeSnakGroup(Collections.singletonList(qualifierSnak));
Claim claim = Datamodel.makeClaim(qid1, baseStatement.getClaim().getMainSnak(),
Collections.singletonList(group));
Statement statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
ItemUpdate update = new ItemUpdateBuilder(qid1).addStatement(statement).build();
assertEquals("Q1377\tP38\tQ865528\tP38\tQ1377\n", export(update));
}
@Test
public void testNoSchema()
throws IOException {
Project project = this.createCSVProject("a,b\nc,d");
Engine engine = new Engine(project);
StringWriter writer = new StringWriter();
Properties properties = new Properties();
exporter.export(project, properties, engine, writer);
assertEquals(QuickStatementsExporter.noSchemaErrorMessage, writer.toString());
}
@Test
public void testGetContentType() {
assertEquals("text/plain", exporter.getContentType());
}
}

57
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java
View File

@ -1,5 +1,31 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.StringReader;
@ -12,9 +38,6 @@ import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
@ -25,11 +48,11 @@ import com.google.refine.util.Pool;
import edu.mit.simile.butterfly.ButterflyModule;
public abstract class OperationTest extends RefineTest {
protected Project project = null;
protected ButterflyModule module = null;
protected Pool pool = null;
@BeforeMethod
public void setUp() {
project = createCSVProject("a,b\nc,d");
@ -37,17 +60,20 @@ public abstract class OperationTest extends RefineTest {
when(module.getName()).thenReturn("wikidata");
pool = new Pool();
}
protected void registerOperation(String name, Class klass) {
OperationRegistry.registerOperation(module, name, klass);
}
public abstract AbstractOperation reconstruct() throws Exception;
public abstract JSONObject getJson() throws Exception;
public abstract AbstractOperation reconstruct()
throws Exception;
public abstract JSONObject getJson()
throws Exception;
@Test
public void testReconstruct() throws Exception {
public void testReconstruct()
throws Exception {
JSONObject json = getJson();
AbstractOperation op = reconstruct();
StringWriter writer = new StringWriter();
@ -55,13 +81,14 @@ public abstract class OperationTest extends RefineTest {
op.write(jsonWriter, new Properties());
JacksonSerializationTest.assertJsonEquals(json.toString(), writer.toString());
}
protected LineNumberReader makeReader(String input) {
StringReader reader = new StringReader(input);
return new LineNumberReader(reader);
}
protected String saveChange(Change change) throws IOException {
protected String saveChange(Change change)
throws IOException {
StringWriter writer = new StringWriter();
change.save(writer, new Properties());
return writer.toString();

53
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.junit.Assert.assertEquals;
@ -14,41 +37,43 @@ import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Recon;
public class PerformWikibaseEditsOperationTest extends OperationTest {
@BeforeMethod
public void registerOperation() {
registerOperation("perform-wikibase-edits", PerformWikibaseEditsOperation.class);
}
@Override
public AbstractOperation reconstruct() throws Exception {
public AbstractOperation reconstruct()
throws Exception {
JSONObject json = getJson();
return PerformWikibaseEditsOperation.reconstruct(project, json);
}
@Override
public JSONObject getJson() throws Exception {
public JSONObject getJson()
throws Exception {
return TestingData.jsonFromFile("data/operations/perform-edits.json");
}
@Test
public void testLoadChange() throws Exception {
String changeString = "newItems={\"qidMap\":{\"1234\":\"Q789\"}}\n" +
"/ec/\n";
public void testLoadChange()
throws Exception {
String changeString = "newItems={\"qidMap\":{\"1234\":\"Q789\"}}\n" + "/ec/\n";
LineNumberReader reader = makeReader(changeString);
Change change = PerformWikibaseEditsOperation.PerformWikibaseEditsChange.load(reader, pool);
project.rows.get(0).cells.set(0, TestingData.makeNewItemCell(1234L, "my new item"));
change.apply(project);
assertEquals(Recon.Judgment.Matched, project.rows.get(0).cells.get(0).recon.judgment);
assertEquals("Q789", project.rows.get(0).cells.get(0).recon.match.id);
change.revert(project);
assertEquals(Recon.Judgment.New, project.rows.get(0).cells.get(0).recon.judgment);
assertEquals(changeString, saveChange(change));
}

53
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.junit.Assert.assertEquals;
@ -14,44 +37,44 @@ import org.testng.annotations.Test;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
public class SaveWikibaseSchemaOperationTest extends OperationTest {
@BeforeMethod
public void registerOperation() {
registerOperation("save-wikibase-schema", SaveWikibaseSchemaOperation.class);
}
@Override
public AbstractOperation reconstruct() throws Exception {
public AbstractOperation reconstruct()
throws Exception {
return SaveWikibaseSchemaOperation.reconstruct(project, getJson());
}
@Override
public JSONObject getJson() throws Exception {
public JSONObject getJson()
throws Exception {
return TestingData.jsonFromFile("data/operations/save-schema.json");
}
@Test
public void testLoadChange() throws Exception {
public void testLoadChange()
throws Exception {
JSONObject schemaJson = TestingData.jsonFromFile("data/schema/inception.json");
String changeString =
"newSchema="+schemaJson.toString()+"\n" +
"oldSchema=\n" +
"/ec/";
String changeString = "newSchema=" + schemaJson.toString() + "\n" + "oldSchema=\n" + "/ec/";
WikibaseSchema schema = WikibaseSchema.reconstruct(schemaJson);
LineNumberReader reader = makeReader(changeString);
Change change = SaveWikibaseSchemaOperation.WikibaseSchemaChange.load(reader, pool);
change.apply(project);
assertEquals(schema, project.overlayModels.get(SaveWikibaseSchemaOperation.WikibaseSchemaChange.overlayModelKey));
assertEquals(schema,
project.overlayModels.get(SaveWikibaseSchemaOperation.WikibaseSchemaChange.overlayModelKey));
change.revert(project);
assertNull(project.overlayModels.get(SaveWikibaseSchemaOperation.WikibaseSchemaChange.overlayModelKey));
saveChange(change); // not checking for equality because JSON serialization varies
}
}

32
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Arrays;
@ -8,14 +31,13 @@ import java.util.stream.Collectors;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class MockConstraintFetcher implements ConstraintFetcher {
public static PropertyIdValue pidWithInverse = Datamodel.makeWikidataPropertyIdValue("P350");
public static PropertyIdValue inversePid = Datamodel.makeWikidataPropertyIdValue("P57");
public static PropertyIdValue allowedQualifierPid = Datamodel.makeWikidataPropertyIdValue("P34");
public static PropertyIdValue mandatoryQualifierPid = Datamodel.makeWikidataPropertyIdValue("P97");
public static PropertyIdValue mainSnakPid = Datamodel.makeWikidataPropertyIdValue("P1234");
public static PropertyIdValue qualifierPid = Datamodel.makeWikidataPropertyIdValue("P987");
public static PropertyIdValue referencePid = Datamodel.makeWikidataPropertyIdValue("P384");
@ -26,8 +48,8 @@ public class MockConstraintFetcher implements ConstraintFetcher {
}
/**
* This constraint is purposely left inconsistent (the inverse
* constraint holds only on one side).
* This constraint is purposely left inconsistent (the inverse constraint holds
* only on one side).
*/
@Override
public PropertyIdValue getInversePid(PropertyIdValue pid) {

39
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/QAWarningStoreTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import static org.junit.Assert.assertEquals;
@ -9,13 +32,13 @@ import org.testng.annotations.Test;
public class QAWarningStoreTest {
public static String exampleJson = "{\"max_severity\":\"CRITICAL\",\"nb_warnings\":5,"
+"\"warnings\":[{\"type\":\"new-item-without-label\",\"bucketId\":null,"
+"\"severity\":\"CRITICAL\",\"count\":3},{\"type\":\"add-statements-with-invalid-format\","
+"\"bucketId\":\"P2427\",\"severity\":\"IMPORTANT\",\"count\":2}]}";
+ "\"warnings\":[{\"type\":\"new-item-without-label\",\"bucketId\":null,"
+ "\"severity\":\"CRITICAL\",\"count\":3},{\"type\":\"add-statements-with-invalid-format\","
+ "\"bucketId\":\"P2427\",\"severity\":\"IMPORTANT\",\"count\":2}]}";
private QAWarningStore store;
private QAWarning otherWarning;
@BeforeMethod
public void setUp() {
store = new QAWarningStore();
@ -24,18 +47,18 @@ public class QAWarningStoreTest {
otherWarning = new QAWarning("new-item-without-label", null, QAWarning.Severity.CRITICAL, 3);
store.addWarning(otherWarning);
}
@Test
public void testSerialize() {
JacksonSerializationTest.testSerialize(store, exampleJson);
}
@Test
public void testCount() {
assertEquals(5, store.getNbWarnings());
assertEquals(2, store.getWarnings().size());
}
@Test
public void testMaxSeverity() {
assertEquals(QAWarning.Severity.CRITICAL, store.getMaxSeverity());

51
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/QAWarningTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import static org.junit.Assert.assertEquals;
@ -6,25 +29,20 @@ import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
public class QAWarningTest {
public static QAWarning exampleWarning = new QAWarning("add-statements-with-invalid-format",
"P2427",
QAWarning.Severity.IMPORTANT,
1);
public static String exampleJson =
"{\"severity\":\"IMPORTANT\","+
"\"count\":1,\"bucketId\":\"P2427\",\"type\":\"add-statements-with-invalid-format\"}";
public static QAWarning exampleWarning = new QAWarning("add-statements-with-invalid-format", "P2427",
QAWarning.Severity.IMPORTANT, 1);
public static String exampleJson = "{\"severity\":\"IMPORTANT\","
+ "\"count\":1,\"bucketId\":\"P2427\",\"type\":\"add-statements-with-invalid-format\"}";
@Test
public void testSerialize() {
JacksonSerializationTest.testSerialize(exampleWarning, exampleJson);
}
@Test
public void testAggregate() {
QAWarning firstWarning = new QAWarning("add-statements-with-invalid-format",
"P2427",
QAWarning.Severity.INFO,
QAWarning firstWarning = new QAWarning("add-statements-with-invalid-format", "P2427", QAWarning.Severity.INFO,
1);
firstWarning.setProperty("foo", "bar");
assertEquals(exampleWarning.getAggregationId(), firstWarning.getAggregationId());
@ -35,13 +53,10 @@ public class QAWarningTest {
assertEquals(exampleWarning.getSeverity(), merged.getSeverity());
assertEquals("bar", merged.getProperties().get("foo"));
}
@Test
public void testCompare() {
QAWarning otherWarning = new QAWarning("no-reference",
"no-reference",
QAWarning.Severity.WARNING,
1);
QAWarning otherWarning = new QAWarning("no-reference", "no-reference", QAWarning.Severity.WARNING, 1);
assertEquals(1, otherWarning.compareTo(exampleWarning));
assertEquals(-1, exampleWarning.compareTo(otherWarning));
assertEquals(0, exampleWarning.compareTo(exampleWarning));

51
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/WikidataConstraintFetcherTests.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import org.testng.Assert;
@ -8,9 +31,9 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import java.util.regex.Pattern;
public class WikidataConstraintFetcherTests {
private ConstraintFetcher fetcher;
private PropertyIdValue headOfGovernment;
private PropertyIdValue startTime;
private PropertyIdValue endTime;
@ -20,7 +43,7 @@ public class WikidataConstraintFetcherTests {
private PropertyIdValue partOf;
private PropertyIdValue referenceURL;
private PropertyIdValue reasonForDeprecation;
public WikidataConstraintFetcherTests() {
fetcher = new WikidataConstraintFetcher();
headOfGovernment = Datamodel.makeWikidataPropertyIdValue("P6");
@ -33,41 +56,41 @@ public class WikidataConstraintFetcherTests {
referenceURL = Datamodel.makeWikidataPropertyIdValue("P854");
reasonForDeprecation = Datamodel.makeWikidataPropertyIdValue("P2241");
}
@Test
public void testGetFormatConstraint() {
String regex = fetcher.getFormatRegex(gridId);
Pattern pattern = Pattern.compile(regex);
Assert.assertTrue(pattern.matcher("grid.470811.b").matches());
Assert.assertFalse(pattern.matcher("501100006367").matches());
Assert.assertNull(fetcher.getFormatRegex(instanceOf));
}
@Test
public void testGetInverseConstraint() {
Assert.assertEquals(fetcher.getInversePid(partOf), hasPart);
}
@Test
public void testOnlyReferences() {
Assert.assertTrue(fetcher.isForReferencesOnly(referenceURL));
Assert.assertFalse(fetcher.isForReferencesOnly(reasonForDeprecation));
}
@Test
public void testOnlyQualifiers() {
Assert.assertTrue(fetcher.isForQualifiersOnly(reasonForDeprecation));
Assert.assertFalse(fetcher.isForQualifiersOnly(headOfGovernment));
}
@Test
public void testOnlyValues() {
Assert.assertTrue(fetcher.isForValuesOnly(headOfGovernment));
Assert.assertFalse(fetcher.isForValuesOnly(referenceURL));
}
@Test
public void testAllowedQualifiers() {
Assert.assertTrue(fetcher.allowedQualifiers(headOfGovernment).contains(startTime));
@ -75,20 +98,20 @@ public class WikidataConstraintFetcherTests {
Assert.assertFalse(fetcher.allowedQualifiers(headOfGovernment).contains(headOfGovernment));
Assert.assertNull(fetcher.allowedQualifiers(startTime));
}
@Test
public void testMandatoryQualifiers() {
Assert.assertTrue(fetcher.mandatoryQualifiers(headOfGovernment).contains(startTime));
Assert.assertFalse(fetcher.mandatoryQualifiers(headOfGovernment).contains(endTime));
Assert.assertNull(fetcher.allowedQualifiers(startTime));
}
@Test
public void testSingleValue() {
Assert.assertFalse(fetcher.hasSingleValue(headOfGovernment));
Assert.assertTrue(fetcher.hasSingleValue(gridId));
}
@Test
public void testDistinctValues() {
Assert.assertFalse(fetcher.hasDistinctValues(partOf));

31
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.testing.TestingData;
@ -17,12 +40,8 @@ public class DistinctValuesScrutinizerTest extends StatementScrutinizerTest {
public void testTrigger() {
ItemIdValue idA = TestingData.existingId;
ItemIdValue idB = TestingData.matchedId;
ItemUpdate updateA = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, idB))
.build();
ItemUpdate updateB = new ItemUpdateBuilder(idB)
.addStatement(TestingData.generateStatement(idB, idB))
.build();
ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).build();
ItemUpdate updateB = new ItemUpdateBuilder(idB).addStatement(TestingData.generateStatement(idB, idB)).build();
scrutinize(updateA, updateB);
assertWarningsRaised(DistinctValuesScrutinizer.type);
}

29
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.testng.annotations.Test;
@ -9,19 +32,19 @@ public class FormatScrutinizerTest extends ValueScrutinizerTest {
public EditScrutinizer getScrutinizer() {
return new FormatScrutinizer();
}
@Test
public void testTrigger() {
scrutinize(Datamodel.makeStringValue("not a number"));
assertWarningsRaised(FormatScrutinizer.type);
}
@Test
public void testNoIssue() {
scrutinize(Datamodel.makeStringValue("1234"));
assertNoWarningRaised();
}
@Test
public void testIncompleteMatch() {
scrutinize(Datamodel.makeStringValue("42 is a number"));

35
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.MockConstraintFetcher;
@ -9,7 +32,7 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class InverseConstaintScrutinizerTest extends StatementScrutinizerTest {
private ItemIdValue idA = TestingData.existingId;
private ItemIdValue idB = TestingData.newIdB;
private PropertyIdValue pidWithInverse = MockConstraintFetcher.pidWithInverse;
@ -19,20 +42,18 @@ public class InverseConstaintScrutinizerTest extends StatementScrutinizerTest {
public EditScrutinizer getScrutinizer() {
return new InverseConstraintScrutinizer();
}
@Test
public void testTrigger() {
ItemUpdate update = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, pidWithInverse, idB))
.build();
.addStatement(TestingData.generateStatement(idA, pidWithInverse, idB)).build();
scrutinize(update);
assertWarningsRaised(InverseConstraintScrutinizer.type);
}
@Test
public void testNoSymmetricClosure() {
ItemUpdate update = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, inversePid, idB))
ItemUpdate update = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, inversePid, idB))
.build();
scrutinize(update);
assertNoWarningRaised();

55
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Collections;
@ -11,58 +34,50 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
public class NewItemScrutinizerTest extends ScrutinizerTest {
private Claim claim = Datamodel.makeClaim(TestingData.newIdA,
Datamodel.makeValueSnak(Datamodel.makeWikidataPropertyIdValue("P31"), TestingData.existingId),
Collections.emptyList());
Collections.emptyList());
private Statement p31Statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
@Override
public EditScrutinizer getScrutinizer() {
return new NewItemScrutinizer();
}
@Test
public void testTrigger() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA).build();
scrutinize(update);
assertWarningsRaised(
NewItemScrutinizer.noDescType,
NewItemScrutinizer.noLabelType,
NewItemScrutinizer.noTypeType,
NewItemScrutinizer.newItemType);
assertWarningsRaised(NewItemScrutinizer.noDescType, NewItemScrutinizer.noLabelType,
NewItemScrutinizer.noTypeType, NewItemScrutinizer.newItemType);
}
@Test
public void testEmptyItem() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.existingId).build();
scrutinize(update);
assertNoWarningRaised();
}
@Test
public void testGoodNewItem() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"))
.addStatement(p31Statement)
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType);
}
@Test
public void testDeletedStatements() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"))
.addStatement(p31Statement)
.deleteStatement(TestingData.generateStatement(TestingData.newIdA,
TestingData.matchedId))
.build();
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.deleteStatement(TestingData.generateStatement(TestingData.newIdA, TestingData.matchedId)).build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType, NewItemScrutinizer.deletedStatementsType);
}

29
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.testing.TestingData;
@ -10,19 +33,19 @@ public class NoEditsMadeScrutinizerTest extends ScrutinizerTest {
public EditScrutinizer getScrutinizer() {
return new NoEditsMadeScrutinizer();
}
@Test
public void testTrigger() {
scrutinize();
assertWarningsRaised(NoEditsMadeScrutinizer.type);
}
@Test
public void testNonNull() {
scrutinize(new ItemUpdateBuilder(TestingData.newIdA).build());
assertNoWarningRaised();
}
@Test
public void testNull() {
scrutinize(new ItemUpdateBuilder(TestingData.existingId).build());

48
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Arrays;
@ -16,6 +39,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
public class QualifierCompatibilityScrutinizerTest extends StatementScrutinizerTest {
private Snak disallowedQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.qualifierPid);
private Snak mandatoryQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.mandatoryQualifierPid);
private Snak allowedQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.allowedQualifierPid);
@ -24,36 +48,36 @@ public class QualifierCompatibilityScrutinizerTest extends StatementScrutinizerT
public EditScrutinizer getScrutinizer() {
return new QualifierCompatibilityScrutinizer();
}
@Test
public void testDisallowedQualifier() {
scrutinize(makeStatement(disallowedQualifier,mandatoryQualifier));
scrutinize(makeStatement(disallowedQualifier, mandatoryQualifier));
assertWarningsRaised(QualifierCompatibilityScrutinizer.disallowedQualifiersType);
}
@Test
public void testMissingQualifier() {
scrutinize(makeStatement());
assertWarningsRaised(QualifierCompatibilityScrutinizer.missingMandatoryQualifiersType);
}
@Test
public void testGoodEdit() {
scrutinize(makeStatement(allowedQualifier,mandatoryQualifier));
scrutinize(makeStatement(allowedQualifier, mandatoryQualifier));
assertNoWarningRaised();
}
private Statement makeStatement(Snak... qualifiers) {
Claim claim = Datamodel.makeClaim(TestingData.existingId,
Claim claim = Datamodel.makeClaim(TestingData.existingId,
Datamodel.makeNoValueSnak(MockConstraintFetcher.mainSnakPid), makeQualifiers(qualifiers));
return Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
}
private List<SnakGroup> makeQualifiers(Snak[] qualifiers) {
List<Snak> snaks = Arrays.asList(qualifiers);
return snaks.stream()
.map((Snak q) -> Datamodel.makeSnakGroup(Collections.<Snak>singletonList(q)))
.collect(Collectors.toList());
List<Snak> snaks = Arrays.asList(qualifiers);
return snaks.stream().map((Snak q) -> Datamodel.makeSnakGroup(Collections.<Snak> singletonList(q)))
.collect(Collectors.toList());
}
}

Some files were not shown because too many files have changed in this diff Show More