s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add license headers, general cleanups on Java files

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-03-04 00:14:59 +00:00
parent d6b229f25e
commit 0b14a1a627
140 changed files with 5494 additions and 2592 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
extensions/wikidata/credits.txt
View File

@ -1,10 +1,32 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2018 Antonin Delpeuch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#-------------------------------------------------------------------------------
Sources for the images:
- Information.svg by User:Bobarino:
The following images were obtained from Wikimedia Commons:
- Information.svg by User:Bobarino: (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information.svg
- Warning.svg by User:Ezekiel63745
- Warning.svg by User:Ezekiel63745 (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information_orange.svg
- Important.svg originally from David Vignoni
- Important.svg originally from David Vignoni (GNU LGPL)
https://commons.wikimedia.org/wiki/File:Nuvola_apps_important.svg
- Critical.svg by User:Stannered
- Critical.svg by User:Stannered (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Stop_x_nuvola.svg

25
extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
@ -14,13 +37,13 @@ import org.openrefine.wikidata.editing.ConnectionManager;
import com.google.refine.commands.Command;
public class LoginCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String username = request.getParameter("wb-username");
String password = request.getParameter("wb-password");
String remember = request.getParameter("remember-credentials");
System.out.println(remember);
ConnectionManager manager = ConnectionManager.getInstance();
if (username != null && password != null) {
manager.login(username, password, "on".equals(remember));

26
extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import javax.servlet.http.HttpServletRequest;
@ -15,8 +38,7 @@ public class PerformWikibaseEditsCommand extends EngineDependentCommand {
protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig)
throws Exception {
String summary = request.getParameter("summary");
return new PerformWikibaseEditsOperation(engineConfig,
summary);
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
}

27
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
/*
Copyright 2010, Google Inc.
@ -34,8 +57,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
@ -63,6 +84,7 @@ import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class PreviewWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -75,7 +97,6 @@ public class PreviewWikibaseSchemaCommand extends Command {
String jsonString = request.getParameter("schema");
WikibaseSchema schema = null;
if (jsonString != null) {
try {

23
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;

47
extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java
View File

@ -1,41 +1,61 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
/**
* Manages a connection to Wikidata, with login credentials stored
* in the preferences.
* Manages a connection to Wikidata, with login credentials stored in the
* preferences.
*
* Ideally, we should store only the cookies and not the password.
* But Wikidata-Toolkit does not allow for that as cookies are kept
* private.
* Ideally, we should store only the cookies and not the password. But
* Wikidata-Toolkit does not allow for that as cookies are kept private.
*
* This class is also hard-coded for Wikidata: generalization to other
* Wikibase instances should be feasible though.
* This class is also hard-coded for Wikidata: generalization to other Wikibase
* instances should be feasible though.
*
* @author antonin
* @author Antonin Delpeuch
*/
public class ConnectionManager {
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
private PreferenceStore prefStore;
private ApiConnection connection;
private static class ConnectionManagerHolder {
private static final ConnectionManager instance = new ConnectionManager();
}
@ -77,8 +97,7 @@ public class ConnectionManager {
if (savedCredentials != null) {
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(savedCredentials.getString("username"),
savedCredentials.getString("password"));
connection.login(savedCredentials.getString("username"), savedCredentials.getString("password"));
} catch (LoginFailedException e) {
connection = null;
} catch (JSONException e) {

78
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
@ -28,8 +51,7 @@ import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
*/
public class EditBatchProcessor {
static final Logger logger = LoggerFactory
.getLogger(EditBatchProcessor.class);
static final Logger logger = LoggerFactory.getLogger(EditBatchProcessor.class);
private WikibaseDataFetcher fetcher;
private WikibaseDataEditor editor;
@ -44,11 +66,10 @@ public class EditBatchProcessor {
private Map<String, EntityDocument> currentDocs;
private int batchSize;
/**
* Initiates the process of pushing a batch of updates
* to Wikibase. This schedules the updates and is a
* prerequisite for calling {@link performOneEdit}.
* Initiates the process of pushing a batch of updates to Wikibase. This
* schedules the updates and is a prerequisite for calling
* {@link performOneEdit}.
*
* @param fetcher
* the fetcher to use to retrieve the current state of items
@ -61,14 +82,11 @@ public class EditBatchProcessor {
* @param summary
* the summary to append to all edits
* @param batchSize
* the number of items that should be retrieved in one go from the API
* the number of items that should be retrieved in one go from the
* API
*/
public EditBatchProcessor(WikibaseDataFetcher fetcher,
WikibaseDataEditor editor,
List<ItemUpdate> updates,
NewItemLibrary library,
String summary,
int batchSize) {
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
NewItemLibrary library, String summary, int batchSize) {
this.fetcher = fetcher;
this.editor = editor;
editor.setEditAsBot(true); // this will not do anything if the user does not
@ -88,13 +106,13 @@ public class EditBatchProcessor {
this.currentDocs = Collections.emptyMap();
}
/**
* Performs the next edit in the batch.
*
* @throws InterruptedException
*/
public void performEdit() throws InterruptedException {
public void performEdit()
throws InterruptedException {
if (remainingEdits() == 0) {
return;
}
@ -116,8 +134,7 @@ public class EditBatchProcessor {
ItemDocument itemDocument = Datamodel.makeItemDocument(update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
update.getAddedStatementGroups(),
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
@ -125,28 +142,12 @@ public class EditBatchProcessor {
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
/*
TermStatementUpdate tsUpdate = new TermStatementUpdate(
currentDocument,
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>()
);
ObjectMapper mapper = new ObjectMapper();
logger.info(mapper.writeValueAsString(update));
logger.info(update.toString());
logger.info(tsUpdate.getJsonUpdateString()); */
editor.updateTermsStatements(currentDocument,
update.getLabels().stream().collect(Collectors.toList()),
editor.updateTermsStatements(currentDocument, update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()),
summary);
update.getDeletedStatements().stream().collect(Collectors.toList()), summary);
}
} catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way
@ -172,7 +173,8 @@ public class EditBatchProcessor {
return (100 * (globalCursor + batchCursor)) / scheduled.size();
}
protected void prepareNewBatch() throws InterruptedException {
protected void prepareNewBatch()
throws InterruptedException {
// remove the previous batch from the remainingUpdates
globalCursor += currentBatch.size();
currentBatch.clear();
@ -183,9 +185,7 @@ public class EditBatchProcessor {
} else {
currentBatch = remainingUpdates.subList(0, batchSize);
}
List<String> qidsToFetch = currentBatch.stream()
.filter(u -> !u.isNew())
.map(u -> u.getItemId().getId())
List<String> qidsToFetch = currentBatch.stream().filter(u -> !u.isNew()).map(u -> u.getItemId().getId())
.collect(Collectors.toList());
// Get the current documents for this batch of updates

81
extensions/wikidata/src/org/openrefine/wikidata/editing/NewItemLibrary.java
View File

@ -1,28 +1,49 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import com.google.refine.model.Project;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
/**
* This keeps track of the new items that we
* have created for each internal reconciliation id.
* This keeps track of the new items that we have created for each internal
* reconciliation id.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class NewItemLibrary {
@ -40,7 +61,9 @@ public class NewItemLibrary {
/**
* Retrieves the Qid allocated to a given new cell
* @param id: the fake ItemId generated by the cell
*
* @param id:
* the fake ItemId generated by the cell
* @return the qid (or null if unallocated yet)
*/
public String getQid(long id) {
@ -49,30 +72,32 @@ public class NewItemLibrary {
/**
* Stores a Qid associated to a new cell
* @param id : the internal reconciliation id of the new cell
* @param qid : the associated Qid returned by Wikibase
*
* @param id
* : the internal reconciliation id of the new cell
* @param qid
* : the associated Qid returned by Wikibase
*/
public void setQid(long id, String qid) {
map.put(id, qid);
}
/**
* Changes the "new" reconciled cells to their allocated
* qids for later use.
* Changes the "new" reconciled cells to their allocated qids for later use.
*
* @param reset: set to true to revert the operation (set cells to "new")
* @param reset:
* set to true to revert the operation (set cells to "new")
*/
public void updateReconciledCells(Project project, boolean reset) {
Set<Integer> impactedColumns = new HashSet<>();
/*
* Note that there is a slight violation of OpenRefine's model here:
* if we reconcile multiple cells to the same new item, and then
* perform this operation on a subset of the corresponding rows,
* we are going to modify cells that are outside the facet (because
* they are reconciled to the same cell). But I think this is the
* right thing to do.
* Note that there is a slight violation of OpenRefine's model here: if we
* reconcile multiple cells to the same new item, and then perform this
* operation on a subset of the corresponding rows, we are going to modify cells
* that are outside the facet (because they are reconciled to the same cell).
* But I think this is the right thing to do.
*/
for (Row row : project.rows) {
@ -82,17 +107,14 @@ public class NewItemLibrary {
continue;
}
Recon recon = cell.recon;
if (Recon.Judgment.New.equals(recon.judgment) && !reset &&
map.containsKey(recon.judgmentHistoryEntry)) {
if (Recon.Judgment.New.equals(recon.judgment) && !reset
&& map.containsKey(recon.judgmentHistoryEntry)) {
recon.judgment = Recon.Judgment.Matched;
recon.match = new ReconCandidate(
map.get(recon.judgmentHistoryEntry),
cell.value.toString(),
new String[0],
100);
recon.match = new ReconCandidate(map.get(recon.judgmentHistoryEntry), cell.value.toString(),
new String[0], 100);
impactedColumns.add(i);
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset &&
map.containsKey(recon.judgmentHistoryEntry)) {
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset
&& map.containsKey(recon.judgmentHistoryEntry)) {
recon.judgment = Recon.Judgment.New;
recon.match = null;
impactedColumns.add(i);
@ -108,6 +130,7 @@ public class NewItemLibrary {
/**
* Getter, only meant to be used by Jackson
*
* @return the underlying map
*/
@JsonProperty("qidMap")

72
extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java
View File

@ -1,33 +1,52 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.helpers.DatamodelConverter;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A class that rewrites an {@link ItemUpdate},
* replacing reconciled entity id values by their concrete
* values after creation of all the new items involved.
* A class that rewrites an {@link ItemUpdate}, replacing reconciled entity id
* values by their concrete values after creation of all the new items involved.
*
* If an item has not been created yet, an {@link IllegalArgumentException}
* will be raised.
* If an item has not been created yet, an {@link IllegalArgumentException} will
* be raised.
*
* The subject is treated as a special case: it is returned unchanged.
* This is because it is guaranteed not to appear in the update (but
* it does appear in the datamodel representation as the subject is passed around
* to the Claim objects its document contains).
* The subject is treated as a special case: it is returned unchanged. This is
* because it is guaranteed not to appear in the update (but it does appear in
* the datamodel representation as the subject is passed around to the Claim
* objects its document contains).
*
* @author Antonin Delpeuch
*
@ -38,9 +57,8 @@ public class ReconEntityRewriter extends DatamodelConverter {
private ItemIdValue subject;
/**
* Constructor. Sets up a rewriter which uses the provided library
* to look up qids of new items, and the subject (which should not be
* rewritten).
* Constructor. Sets up a rewriter which uses the provided library to look up
* qids of new items, and the subject (which should not be rewritten).
*
* @param library
* @param subject
@ -64,25 +82,21 @@ public class ReconEntityRewriter extends DatamodelConverter {
throw new IllegalArgumentException(
"Trying to rewrite an update where a new item was not created yet.");
}
return Datamodel.makeItemIdValue(newId,
recon.getRecon().identifierSpace);
return Datamodel.makeItemIdValue(newId, recon.getRecon().identifierSpace);
}
}
return super.copy(value);
}
public ItemUpdate rewrite(ItemUpdate update) {
Set<MonolingualTextValue> labels = update.getLabels().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream()
.map(l -> copy(l)).collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream()
.map(l -> copy(l)).collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements,
deletedStatements, labels, descriptions, aliases);
Set<MonolingualTextValue> labels = update.getLabels().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream().map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream().map(l -> copy(l))
.collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream().map(l -> copy(l))
.collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

37
extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Properties;
@ -9,10 +32,10 @@ import org.json.JSONWriter;
import com.google.refine.Jsonizable;
/**
* This is just the necessary bits to store Wikidata credentials
* in OpenRefine's preference store.
* This is just the necessary bits to store Wikidata credentials in OpenRefine's
* preference store.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
class WikibaseCredentials implements Jsonizable {
@ -55,11 +78,9 @@ class WikibaseCredentials implements Jsonizable {
writer.endObject();
}
public static WikibaseCredentials load(JSONObject obj) throws JSONException {
return new WikibaseCredentials(
obj.getString("username"),
obj.getString("password"));
public static WikibaseCredentials load(JSONObject obj)
throws JSONException {
return new WikibaseCredentials(obj.getString("username"), obj.getString("password"));
}
}

69
extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.math.BigDecimal;
@ -14,14 +37,12 @@ import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* Prints a Wikibase value as a string as required by QuickStatements.
* Format documentation:
* https://www.wikidata.org/wiki/Help:QuickStatements
* Prints a Wikibase value as a string as required by QuickStatements. Format
* documentation: https://www.wikidata.org/wiki/Help:QuickStatements
*
* Any new entity id will be
* assumed to be the last one created, represented with "LAST". It is
* fine to do this assumption because we are working on edit batches
* previously scheduled by {@link QuickStatementsUpdateScheduler}.
* Any new entity id will be assumed to be the last one created, represented
* with "LAST". It is fine to do this assumption because we are working on edit
* batches previously scheduled by {@link QuickStatementsUpdateScheduler}.
*
* @author Antonin Delpeuch
*
@ -45,19 +66,12 @@ public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(GlobeCoordinatesValue value) {
return String.format(
Locale.US,
"@%f/%f",
value.getLatitude(),
value.getLongitude());
return String.format(Locale.US, "@%f/%f", value.getLatitude(), value.getLongitude());
}
@Override
public String visit(MonolingualTextValue value) {
return String.format(
"%s:\"%s\"",
value.getLanguageCode(),
value.getText());
return String.format("%s:\"%s\"", value.getLanguageCode(), value.getText());
}
@Override
@ -66,22 +80,16 @@ public class QSValuePrinter implements ValueVisitor<String> {
String unitIri = value.getUnit();
String unitRepresentation = "", boundsRepresentation = "";
if (!unitIri.isEmpty()) {
if (!unitIri.startsWith(unitPrefix))
return null; // QuickStatements only accepts Qids as units
if (!unitIri.startsWith(unitPrefix)) return null; // QuickStatements only accepts Qids as units
unitRepresentation = "U" + unitIri.substring(unitPrefix.length());
}
if (value.getLowerBound() != null) {
// bounds are always null at the same time so we know they are both not null
BigDecimal lowerBound = value.getLowerBound();
BigDecimal upperBound = value.getUpperBound();
boundsRepresentation = String.format(Locale.US, "[%s,%s]",
lowerBound.toString(), upperBound.toString());
boundsRepresentation = String.format(Locale.US, "[%s,%s]", lowerBound.toString(), upperBound.toString());
}
return String.format(
Locale.US,
"%s%s%s",
value.getNumericValue().toString(),
boundsRepresentation,
return String.format(Locale.US, "%s%s%s", value.getNumericValue().toString(), boundsRepresentation,
unitRepresentation);
}
@ -92,14 +100,7 @@ public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(TimeValue value) {
return String.format(
"+%04d-%02d-%02dT%02d:%02d:%02dZ/%d",
value.getYear(),
value.getMonth(),
value.getDay(),
value.getHour(),
value.getMinute(),
value.getSecond(),
value.getPrecision());
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
}
}

52
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.io.IOException;
@ -30,10 +53,8 @@ public class QuickStatementsExporter implements WriterExporter {
final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
public static final String impossibleSchedulingErrorMessage =
"This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage =
"No schema was provided. You need to align your project with Wikidata first.";
public static final String impossibleSchedulingErrorMessage = "This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage = "No schema was provided. You need to align your project with Wikidata first.";
public QuickStatementsExporter() {
}
@ -67,12 +88,14 @@ public class QuickStatementsExporter implements WriterExporter {
* the writer to which the QS should be written
* @throws IOException
*/
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer) throws IOException {
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer)
throws IOException {
List<ItemUpdate> items = schema.evaluate(project, engine);
translateItemList(items, writer);
}
public void translateItemList(List<ItemUpdate> updates, Writer writer) throws IOException {
public void translateItemList(List<ItemUpdate> updates, Writer writer)
throws IOException {
QuickStatementsUpdateScheduler scheduler = new QuickStatementsUpdateScheduler();
try {
List<ItemUpdate> scheduled = scheduler.schedule(updates);
@ -85,7 +108,9 @@ public class QuickStatementsExporter implements WriterExporter {
}
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id, Writer writer) throws IOException {
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id,
Writer writer)
throws IOException {
for (MonolingualTextValue value : values) {
writer.write(qid + "\t");
writer.write(prefix);
@ -96,7 +121,8 @@ public class QuickStatementsExporter implements WriterExporter {
}
}
protected void translateItem(ItemUpdate item, Writer writer) throws IOException {
protected void translateItem(ItemUpdate item, Writer writer)
throws IOException {
String qid = item.getItemId().getId();
if (item.isNew()) {
writer.write("CREATE\n");
@ -116,7 +142,8 @@ public class QuickStatementsExporter implements WriterExporter {
}
}
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer) throws IOException {
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer)
throws IOException {
Claim claim = statement.getClaim();
Value val = claim.getValue();
@ -140,13 +167,15 @@ public class QuickStatementsExporter implements WriterExporter {
}
}
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer) throws IOException {
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer)
throws IOException {
for (Snak s : sg.getSnaks()) {
translateSnak(s, reference, writer);
}
}
protected void translateSnak(Snak s, boolean reference, Writer writer) throws IOException {
protected void translateSnak(Snak s, boolean reference, Writer writer)
throws IOException {
String pid = s.getPropertyId().getId();
if (reference) {
pid = pid.replace('P', 'S');
@ -159,5 +188,4 @@ public class QuickStatementsExporter implements WriterExporter {
}
}
}

84
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java
View File

@ -1,44 +1,48 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import org.openrefine.wikidata.editing.EditBatchProcessor;
import org.openrefine.wikidata.editing.NewItemLibrary;
import org.openrefine.wikidata.editing.ReconEntityRewriter;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.UpdateScheduler;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.util.WebResourceFetcherImpl;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.TermStatementUpdate;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -53,16 +57,13 @@ import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
import com.google.refine.util.Pool;
public class PerformWikibaseEditsOperation extends EngineDependentOperation {
static final Logger logger = LoggerFactory
.getLogger(PerformWikibaseEditsOperation.class);
static final Logger logger = LoggerFactory.getLogger(PerformWikibaseEditsOperation.class);
private String summary;
public PerformWikibaseEditsOperation(
JSONObject engineConfig,
String summary) {
public PerformWikibaseEditsOperation(JSONObject engineConfig, String summary) {
super(engineConfig);
this.summary = summary;
@ -76,12 +77,9 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
if (obj.has("summary")) {
summary = obj.getString("summary");
}
return new PerformWikibaseEditsOperation(
engineConfig,
summary);
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
@ -103,13 +101,9 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
}
@Override
public Process createProcess(Project project, Properties options) throws Exception {
return new PerformEditsProcess(
project,
createEngine(project),
getBriefDescription(project),
summary
);
public Process createProcess(Project project, Properties options)
throws Exception {
return new PerformEditsProcess(project, createEngine(project), getBriefDescription(project), summary);
}
static public class PerformWikibaseEditsChange implements Change {
@ -171,8 +165,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
protected String _summary;
protected final long _historyEntryID;
protected PerformEditsProcess(Project project,
Engine engine, String description, String summary) {
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
super(description);
this._project = project;
this._engine = engine;
@ -199,8 +192,8 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
// Prepare the edits
NewItemLibrary newItemLibrary = new NewItemLibrary();
EditBatchProcessor processor = new EditBatchProcessor(wbdf,
wbde, itemDocuments, newItemLibrary, _summary, 50);
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, _summary,
50);
// Perform edits
logger.info("Performing edits");
@ -221,13 +214,8 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
if (!_canceled) {
Change change = new PerformWikibaseEditsChange(newItemLibrary);
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
PerformWikibaseEditsOperation.this,
change
);
HistoryEntry historyEntry = new HistoryEntry(_historyEntryID, _project, _description,
PerformWikibaseEditsOperation.this, change);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);

53
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java
View File

@ -1,24 +1,38 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WbItemConstant;
import org.openrefine.wikidata.schema.WbItemDocumentExpr;
import org.openrefine.wikidata.schema.WbNameDescExpr;
import org.openrefine.wikidata.schema.WbStatementGroupExpr;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -34,13 +48,11 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj
.getJSONObject("schema")));
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
@ -62,17 +74,17 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
}
@Override
protected HistoryEntry createHistoryEntry(Project project,
long historyEntryID) throws Exception {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID)
throws Exception {
String description = "Save Wikibase schema skeleton";
Change change = new WikibaseSchemaChange(_schema);
return new HistoryEntry(historyEntryID, project, description,
SaveWikibaseSchemaOperation.this, change);
return new HistoryEntry(historyEntryID, project, description, SaveWikibaseSchemaOperation.this, change);
}
static public class WikibaseSchemaChange implements Change {
final protected WikibaseSchema _newSchema;
protected WikibaseSchema _oldSchema = null;
public final static String overlayModelKey = "wikibaseSchema";
@ -98,7 +110,8 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
}
}
public void save(Writer writer, Properties options) throws IOException {
public void save(Writer writer, Properties options)
throws IOException {
writer.write("newSchema=");
writeWikibaseSchema(_newSchema, writer);
writer.write('\n');
@ -120,11 +133,9 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}

42
extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java
View File

@ -1,4 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Set;
@ -14,16 +36,20 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public interface ConstraintFetcher {
/**
* Retrieves the regular expression for formatting a property, or null if
* there is no such constraint
* Retrieves the regular expression for formatting a property, or null if there
* is no such constraint
*
* @param pid
* @return the expression of a regular expression which should be compatible with java.util.regex
* @return the expression of a regular expression which should be compatible
* with java.util.regex
*/
String getFormatRegex(PropertyIdValue pid);
/**
* Retrieves the property that is the inverse of a given property
* @param pid: the property to retrieve the inverse for
*
* @param pid:
* the property to retrieve the inverse for
* @return the pid of the inverse property
*/
PropertyIdValue getInversePid(PropertyIdValue pid);
@ -44,12 +70,14 @@ public interface ConstraintFetcher {
boolean isForReferencesOnly(PropertyIdValue pid);
/**
* Get the list of allowed qualifiers (as property ids) for this property (null if any)
* Get the list of allowed qualifiers (as property ids) for this property (null
* if any)
*/
Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid);
/**
* Get the list of mandatory qualifiers (as property ids) for this property (null if any)
* Get the list of mandatory qualifiers (as property ids) for this property
* (null if any)
*/
Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid);

33
extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
@ -18,8 +41,6 @@ import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.WhitespaceScrutinizer;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.UpdateScheduler;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -30,6 +51,7 @@ import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
*
*/
public class EditInspector {
private Map<String, EditScrutinizer> scrutinizers;
private QAWarningStore warningStore;
private ConstraintFetcher fetcher;
@ -55,6 +77,7 @@ public class EditInspector {
/**
* Adds a new scrutinizer to the inspector
*
* @param scrutinizer
*/
public void register(EditScrutinizer scrutinizer) {
@ -64,9 +87,9 @@ public class EditInspector {
scrutinizer.setFetcher(fetcher);
}
/**
* Inspect a batch of edits with the registered scrutinizers
*
* @param editBatch
*/
public void inspect(List<ItemUpdate> editBatch) {
@ -81,10 +104,8 @@ public class EditInspector {
scrutinizer.scrutinize(mergedUpdates);
}
if (warningStore.getNbWarnings() == 0) {
warningStore.addWarning(new QAWarning(
"no-issue-detected", null, QAWarning.Severity.INFO, 0));
warningStore.addWarning(new QAWarning("no-issue-detected", null, QAWarning.Severity.INFO, 0));
}
}
}

60
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarning.java
View File

@ -1,23 +1,43 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to represent a QA warning emitted by the Wikidata schema
* This could probably be reused at a broader scale, for instance for
* Data Package validation.
* A class to represent a QA warning emitted by the Wikidata schema This could
* probably be reused at a broader scale, for instance for Data Package
* validation.
*
* @author Antonin Delpeuch
*
@ -27,13 +47,15 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
public enum Severity {
INFO, // We just report something to the user but it is probably fine
WARNING, // Edits that look wrong but in some cases they are actually fine
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we might want to allow it
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we
// might want to allow it
CRITICAL, // We should never edit if there is a critical issue
}
/// The type of QA warning emitted
private final String type;
// The key for aggregation of other QA warnings together - this specializes the id
// The key for aggregation of other QA warnings together - this specializes the
// id
private final String bucketId;
// The severity of the issue
private final Severity severity;
@ -66,6 +88,7 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
/**
* Aggregates another QA warning of the same aggregation id.
*
* @param other
*/
public QAWarning aggregate(QAWarning other) {
@ -75,8 +98,7 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
if (other.getSeverity().compareTo(severity) > 0) {
newSeverity = other.getSeverity();
}
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity,
newCount);
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity, newCount);
for (Entry<String, Object> entry : properties.entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
@ -87,10 +109,12 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
}
/**
* Sets a property of the QA warning, to be used by the front-end
* for display.
* @param key: the name of the property
* @param value should be Jackson-serializable
* Sets a property of the QA warning, to be used by the front-end for display.
*
* @param key:
* the name of the property
* @param value
* should be Jackson-serializable
*/
public void setProperty(String key, Object value) {
this.properties.put(key, value);
@ -136,10 +160,8 @@ public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning
return false;
}
QAWarning otherWarning = (QAWarning) other;
return type.equals(otherWarning.getType()) &&
bucketId.equals(otherWarning.getBucketId()) &&
severity.equals(otherWarning.getSeverity()) &&
count == otherWarning.getCount() &&
properties.equals(otherWarning.getProperties());
return type.equals(otherWarning.getType()) && bucketId.equals(otherWarning.getBucketId())
&& severity.equals(otherWarning.getSeverity()) && count == otherWarning.getCount()
&& properties.equals(otherWarning.getProperties());
}
}

24
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarningStore.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
@ -30,6 +53,7 @@ public class QAWarningStore {
/**
* Stores a warning, aggregating it with any existing
*
* @param warning
*/
public void addWarning(QAWarning warning) {

69
extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java
View File

@ -1,16 +1,36 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openrefine.wikidata.utils.EntityCache;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
@ -21,13 +41,14 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* This class provides an abstraction over the way constraint
* definitions are stored in Wikidata.
* This class provides an abstraction over the way constraint definitions are
* stored in Wikidata.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WikidataConstraintFetcher implements ConstraintFetcher {
public static String WIKIDATA_CONSTRAINT_PID = "P2302";
public static String FORMAT_CONSTRAINT_QID = "Q21502404";
@ -55,7 +76,6 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
public static String TYPE_CONSTRAINT_QID = "Q21503250";
@Override
public String getFormatRegex(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, FORMAT_CONSTRAINT_QID);
@ -129,11 +149,15 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
}
/**
* Returns a single constraint for a particular type and a property, or null
* if there is no such constraint
* @param pid: the property to retrieve the constraints for
* @param qid: the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not exist
* Returns a single constraint for a particular type and a property, or null if
* there is no such constraint
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not
* exist
*/
protected List<SnakGroup> getSingleConstraint(PropertyIdValue pid, String qid) {
Statement statement = getConstraintsByType(pid, qid).findFirst().orElse(null);
@ -145,20 +169,24 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
/**
* Gets the list of constraints of a particular type for a property
* @param pid: the property to retrieve the constraints for
* @param qid: the type of the constraints
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the stream of matching constraint statements
*/
protected Stream<Statement> getConstraintsByType(PropertyIdValue pid, String qid) {
Stream<Statement> allConstraints = getConstraintStatements(pid)
.stream()
Stream<Statement> allConstraints = getConstraintStatements(pid).stream()
.filter(s -> ((EntityIdValue) s.getValue()).getId().equals(qid));
return allConstraints;
}
/**
* Gets all the constraint statements for a given property
* @param pid : the id of the property to retrieve the constraints for
*
* @param pid
* : the id of the property to retrieve the constraints for
* @return the list of constraint statements
*/
protected List<Statement> getConstraintStatements(PropertyIdValue pid) {
@ -173,8 +201,11 @@ public class WikidataConstraintFetcher implements ConstraintFetcher {
/**
* Returns the values of a given property in qualifiers
* @param groups: the qualifiers
* @param pid: the property to filter on
*
* @param groups:
* the qualifiers
* @param pid:
* the property to filter on
* @return
*/
protected List<Value> findValues(List<SnakGroup> groups, String pid) {

35
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java
View File

@ -1,9 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -12,8 +33,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for properties using the same value
* on different items.
* A scrutinizer that checks for properties using the same value on different
* items.
*
* @author Antonin Delpeuch
*
@ -40,11 +61,7 @@ public class DistinctValuesScrutinizer extends StatementScrutinizer {
}
if (seen.containsKey(mainSnakValue)) {
EntityIdValue otherId = seen.get(mainSnakValue);
QAWarning issue = new QAWarning(
type,
pid.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("item1_entity", entityId);
issue.setProperty("item2_entity", otherId);

32
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java
View File

@ -1,8 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
import org.openrefine.wikidata.qa.WikidataConstraintFetcher;
import org.openrefine.wikidata.qa.ConstraintFetcher;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarning.Severity;
@ -34,7 +56,9 @@ public abstract class EditScrutinizer {
/**
* Reads the candidate edits and emits warnings in the store
* @param edit: the list of ItemUpdates to scrutinize
*
* @param edit:
* the list of ItemUpdates to scrutinize
*/
public abstract void scrutinize(List<ItemUpdate> edit);
@ -48,6 +72,7 @@ public abstract class EditScrutinizer {
/**
* Helper to be used by subclasses to emit simple INFO warnings
*
* @param warning
*/
protected void info(String type) {
@ -57,6 +82,7 @@ public abstract class EditScrutinizer {
/**
* Helper to be used by subclasses to emit simple warnings
*
* @param warning
*/
protected void warning(String type) {
@ -65,6 +91,7 @@ public abstract class EditScrutinizer {
/**
* Helper to be used by subclasses to emit simple important warnings
*
* @param warning
*/
protected void important(String type) {
@ -73,6 +100,7 @@ public abstract class EditScrutinizer {
/**
* Helper to be used by subclasses to emit simple critical warnings
*
* @param warning
*/
protected void critical(String type) {

43
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java
View File

@ -1,8 +1,30 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
@ -12,8 +34,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
/**
* A scrutinizer that detects incorrect formats in text values
* (mostly identifiers).
* A scrutinizer that detects incorrect formats in text values (mostly
* identifiers).
*
* @author Antonin Delpeuch
*
@ -29,10 +51,11 @@ public class FormatScrutinizer extends SnakScrutinizer {
}
/**
* Loads the regex for a property and compiles it to a pattern
* (this is cached upstream, plus we are doing it only once per
* property and batch).
* @param pid the id of the property to fetch the constraints for
* Loads the regex for a property and compiles it to a pattern (this is cached
* upstream, plus we are doing it only once per property and batch).
*
* @param pid
* the id of the property to fetch the constraints for
* @return
*/
protected Pattern getPattern(PropertyIdValue pid) {
@ -60,11 +83,7 @@ public class FormatScrutinizer extends SnakScrutinizer {
}
if (!pattern.matcher(value).matches()) {
if (added) {
QAWarning issue = new QAWarning(
type,
pid.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("regex", pattern.toString());
issue.setProperty("example_value", value);

31
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -14,8 +37,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for missing inverse statements in
* edit batches.
* A scrutinizer that checks for missing inverse statements in edit batches.
*
* @author Antonin Delpeuch
*
@ -85,10 +107,7 @@ public class InverseConstraintScrutinizer extends StatementScrutinizer {
PropertyIdValue missingProperty = propertyPair.getValue();
Set<EntityIdValue> reciprocalLinks = _statements.get(missingProperty).get(idValue);
if (reciprocalLinks == null || !reciprocalLinks.contains(itemLinks.getKey())) {
QAWarning issue = new QAWarning(type,
ourProperty.getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning(type, ourProperty.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("added_property_entity", ourProperty);
issue.setProperty("inverse_property_entity", missingProperty);
issue.setProperty("source_entity", itemLinks.getKey());

32
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
@ -17,15 +40,16 @@ public abstract class ItemUpdateScrutinizer extends EditScrutinizer {
}
/**
* Method to be overridden by subclasses to scrutinize
* an individual item update.
* Method to be overridden by subclasses to scrutinize an individual item
* update.
*
* @param update
*/
public abstract void scrutinize(ItemUpdate update);
/**
* Method to be overridden by subclasses to emit warnings
* once a batch has been completely analyzed.
* Method to be overridden by subclasses to emit warnings once a batch has been
* completely analyzed.
*/
public void batchIsFinished() {
;

47
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
@ -23,31 +46,19 @@ public class NewItemScrutinizer extends ItemUpdateScrutinizer {
info(newItemType);
if (update.getLabels().isEmpty() && update.getAliases().isEmpty()) {
QAWarning issue = new QAWarning(
noLabelType,
null,
QAWarning.Severity.CRITICAL,
1);
QAWarning issue = new QAWarning(noLabelType, null, QAWarning.Severity.CRITICAL, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (update.getDescriptions().isEmpty()) {
QAWarning issue = new QAWarning(
noDescType,
null,
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(noDescType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (!update.getDeletedStatements().isEmpty()) {
QAWarning issue = new QAWarning(
deletedStatementsType,
null,
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(deletedStatementsType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
@ -62,11 +73,7 @@ public class NewItemScrutinizer extends ItemUpdateScrutinizer {
}
}
if (!typeFound) {
QAWarning issue = new QAWarning(
noTypeType,
null,
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(noTypeType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}

24
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java
View File

@ -1,10 +1,32 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
import org.openrefine.wikidata.updates.ItemUpdate;
public class NoEditsMadeScrutinizer extends EditScrutinizer {
public static final String type = "no-edit-generated";

53
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -12,8 +35,8 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that checks the compatibility of the qualifiers
* and the property of a statement, and looks for mandatory qualifiers.
* A scrutinizer that checks the compatibility of the qualifiers and the
* property of a statement, and looks for mandatory qualifiers.
*
* @author Antonin Delpeuch
*/
@ -58,31 +81,25 @@ public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue statementProperty = statement.getClaim().getMainSnak().getPropertyId();
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().
stream().map(e -> e.getProperty()).collect(Collectors.toSet());
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().stream().map(e -> e.getProperty())
.collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty)
.stream().filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers
.stream().filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty).stream()
.filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers.stream()
.filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
for (PropertyIdValue missing : missingQualifiers) {
QAWarning issue = new QAWarning(
missingMandatoryQualifiersType,
statementProperty.getId()+"-"+missing.getId(),
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(missingMandatoryQualifiersType,
statementProperty.getId() + "-" + missing.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("missing_property_entity", missing);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
for (PropertyIdValue disallowed : disallowedQualifiers) {
QAWarning issue = new QAWarning(
disallowedQualifiersType,
statementProperty.getId()+"-"+disallowed.getId(),
QAWarning.Severity.WARNING,
1);
QAWarning issue = new QAWarning(disallowedQualifiersType,
statementProperty.getId() + "-" + disallowed.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("disallowed_property_entity", disallowed);
issue.setProperty("example_item_entity", entityId);

38
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedPositionScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
@ -13,13 +36,10 @@ import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class RestrictedPositionScrutinizer extends StatementScrutinizer {
protected enum SnakPosition {
MAINSNAK,
QUALIFIER,
REFERENCE
MAINSNAK, QUALIFIER, REFERENCE
}
private Map<PropertyIdValue, SnakPosition> _restrictedPids;
@ -70,7 +90,8 @@ public class RestrictedPositionScrutinizer extends StatementScrutinizer {
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position, boolean added) {
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position,
boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, position, added);
@ -83,11 +104,8 @@ public class RestrictedPositionScrutinizer extends StatementScrutinizer {
String positionStr = position.toString().toLowerCase();
String restrictionStr = restriction.toString().toLowerCase();
QAWarning issue = new QAWarning(
"property-restricted-to-"+restrictionStr+"-found-in-"+positionStr,
snak.getPropertyId().getId(),
QAWarning.Severity.IMPORTANT,
1);
QAWarning issue = new QAWarning("property-restricted-to-" + restrictionStr + "-found-in-" + positionStr,
snak.getPropertyId().getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", snak.getPropertyId());
addIssue(issue);
}

33
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
@ -5,10 +28,10 @@ import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
/**
* A scrutinizer that checks for self-referential statements.
* These statements are flagged by Wikibase as suspicious.
* A scrutinizer that checks for self-referential statements. These statements
* are flagged by Wikibase as suspicious.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class SelfReferentialScrutinizer extends SnakScrutinizer {
@ -18,9 +41,7 @@ public class SelfReferentialScrutinizer extends SnakScrutinizer {
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (entityId.equals(snak.getValue())) {
QAWarning issue = new QAWarning(
type, null,
QAWarning.Severity.WARNING, 1);
QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", entityId);
addIssue(issue);
}

31
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashSet;
@ -9,8 +32,8 @@ import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* For now this scrutinizer only checks for uniqueness at
* the item level (it ignores qualifiers and references).
* For now this scrutinizer only checks for uniqueness at the item level (it
* ignores qualifiers and references).
*
* @author Antonin Delpeuch
*
@ -27,9 +50,7 @@ public class SingleValueScrutinizer extends ItemUpdateScrutinizer {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (seenSingleProperties.contains(pid)) {
QAWarning issue = new QAWarning(
type, pid.getId(),
QAWarning.Severity.WARNING, 1);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);

37
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SnakScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Iterator;
@ -8,8 +31,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that inspects snaks individually, no matter whether they
* appear as main snaks, qualifiers or references.
* A scrutinizer that inspects snaks individually, no matter whether they appear
* as main snaks, qualifiers or references.
*
* @author Antonin Delpeuch
*
@ -18,9 +41,13 @@ public abstract class SnakScrutinizer extends StatementScrutinizer {
/**
* This is the method that subclasses should override to implement their checks.
* @param snak: the snak to inspect
* @param entityId: the item on which it is going to (dis)appear
* @param added: whether this snak is going to be added or deleted
*
* @param snak:
* the snak to inspect
* @param entityId:
* the item on which it is going to (dis)appear
* @param added:
* whether this snak is going to be added or deleted
*/
public abstract void scrutinize(Snak snak, EntityIdValue entityId, boolean added);

37
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
@ -18,11 +41,15 @@ public abstract class StatementScrutinizer extends ItemUpdateScrutinizer {
}
/**
* The method that should be overridden by subclasses, implementing
* the checks on one statement
* @param statement: the statement to scrutinize
* @param entityId: the id of the entity on which this statement is made or removed
* @param added: whether this statement was added or deleted
* The method that should be overridden by subclasses, implementing the checks
* on one statement
*
* @param statement:
* the statement to scrutinize
* @param entityId:
* the id of the entity on which this statement is made or removed
* @param added:
* whether this statement was added or deleted
*/
public abstract void scrutinize(Statement statement, EntityIdValue entityId, boolean added);
}

25
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -6,7 +29,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer checking for unsourced statements
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class UnsourcedScrutinizer extends StatementScrutinizer {

26
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
@ -8,7 +31,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that inspects the values of snaks and terms
* @author antonin
*
* @author Antonin Delpeuch
*
*/
public abstract class ValueScrutinizer extends SnakScrutinizer {

23
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/WhitespaceScrutinizer.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;

48
extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -10,13 +33,14 @@ import com.google.refine.model.ColumnModel;
import com.google.refine.model.Row;
/**
* A class holding all the necessary information about
* the context in which a schema expression is evaluated.
* A class holding all the necessary information about the context in which a
* schema expression is evaluated.
*
* @author Antonin Delpeuch
*
*/
public class ExpressionContext {
private String baseIRI;
private int rowId;
private Row row;
@ -25,6 +49,7 @@ public class ExpressionContext {
/**
* Builds an expression context to evaluate a schema on a row
*
* @param baseIRI
* the siteIRI of the schema
* @param rowId
@ -34,15 +59,10 @@ public class ExpressionContext {
* @param columnModel
* lets us access cells by column name
* @param warningStore
* where to store the issues encountered when
* evaluating (can be set to null if these issues should be ignored)
* where to store the issues encountered when evaluating (can be set
* to null if these issues should be ignored)
*/
public ExpressionContext(
String baseIRI,
int rowId,
Row row,
ColumnModel columnModel,
QAWarningStore warningStore) {
public ExpressionContext(String baseIRI, int rowId, Row row, ColumnModel columnModel, QAWarningStore warningStore) {
Validate.notNull(baseIRI);
this.baseIRI = baseIRI;
this.rowId = rowId;
@ -56,14 +76,14 @@ public class ExpressionContext {
public String getBaseIRI() {
return baseIRI;
}
/**
* Retrieves a cell in the current row, by column name.
* If the column does not exist, null is returned.
* Retrieves a cell in the current row, by column name. If the column does not
* exist, null is returned.
*
* @param name
* the name of the column to retrieve the cell from
* @return
* the cell
* @return the cell
*/
public Cell getCellByName(String name) {
Column column = columnModel.getColumnByName(name);

81
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java
View File

@ -1,12 +1,35 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -19,8 +42,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
/**
* A constant for a time value, accepting a number of formats
* which determine the precision of the parsed value.
* A constant for a time value, accepting a number of formats which determine
* the precision of the parsed value.
*
* @author Antonin Delpeuch
*
@ -28,32 +51,28 @@ import com.google.common.collect.ImmutableMap;
public class WbDateConstant implements WbExpression<TimeValue> {
/**
* Map of formats accepted by the parser. Each format is associated
* to the time precision it induces (an integer according to Wikibase's data model).
* Map of formats accepted by the parser. Each format is associated to the time
* precision it induces (an integer according to Wikibase's data model).
*/
public static Map<SimpleDateFormat, Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat, Integer> builder()
.put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy"), 9).put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11).put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14)
.build();
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build();
private TimeValue parsed;
private String origDatestamp;
/**
* Constructor. Used for deserialization from JSON.
* The object will be constructed even if the time cannot
* be parsed (it will evaluate to null) in {@link evaluate}.
* Constructor. Used for deserialization from JSON. The object will be
* constructed even if the time cannot be parsed (it will evaluate to null) in
* {@link evaluate}.
*
* @param origDatestamp
* the date value as a string
*/
@JsonCreator
public WbDateConstant(
@JsonProperty("value") String origDatestamp) {
public WbDateConstant(@JsonProperty("value") String origDatestamp) {
Validate.notNull(origDatestamp);
this.setOrigDatestamp(origDatestamp);
}
@ -65,8 +84,8 @@ public class WbDateConstant implements WbExpression<TimeValue> {
}
/**
* Parses a timestamp into a Wikibase {@link TimeValue}. The
* precision is automatically inferred from the format.
* Parses a timestamp into a Wikibase {@link TimeValue}. The precision is
* automatically inferred from the format.
*
* @param datestamp
* the time to parse
@ -74,7 +93,8 @@ public class WbDateConstant implements WbExpression<TimeValue> {
* @throws ParseException
* if the time cannot be parsed
*/
public static TimeValue parse(String datestamp) throws ParseException {
public static TimeValue parse(String datestamp)
throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) {
@ -94,18 +114,13 @@ public class WbDateConstant implements WbExpression<TimeValue> {
Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance();
calendar.setTime(date);
return Datamodel.makeTimeValue(
calendar.get(Calendar.YEAR),
(byte) (calendar.get(Calendar.MONTH)+1), // java starts at 0
(byte) calendar.get(Calendar.DAY_OF_MONTH),
(byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE),
(byte) calendar.get(Calendar.SECOND),
(byte) precision,
0,
1,
calendar.getTimeZone().getRawOffset()/3600000,
TimeValue.CM_GREGORIAN_PRO);
return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), // java
// starts
// at
// 0
(byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 1,
calendar.getTimeZone().getRawOffset() / 3600000, TimeValue.CM_GREGORIAN_PRO);
}
}

27
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -10,8 +33,8 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* An expression that represents a time value, extracted from a string.
* A number of formats are recognized, see {@link WbDateConstant} for details.
* An expression that represents a time value, extracted from a string. A number
* of formats are recognized, see {@link WbDateConstant} for details.
*
* @author Antonin Delpeuch
*

46
extensions/wikidata/src/org/openrefine/wikidata/schema/WbExpression.java
View File

@ -1,20 +1,40 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The base interface for all expressions, which evaluate to a
* particular type T in an ExpressionContext.
* The base interface for all expressions, which evaluate to a particular type T
* in an ExpressionContext.
*/
@JsonTypeInfo(use=JsonTypeInfo.Id.NAME,
include=JsonTypeInfo.As.PROPERTY,
property="type")
@JsonSubTypes({
@Type(value = WbStringConstant.class, name = "wbstringconstant"),
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
@JsonSubTypes({ @Type(value = WbStringConstant.class, name = "wbstringconstant"),
@Type(value = WbStringVariable.class, name = "wbstringvariable"),
@Type(value = WbLocationConstant.class, name = "wblocationconstant"),
@Type(value = WbLocationVariable.class, name = "wblocationvariable"),
@ -28,13 +48,13 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
@Type(value = WbPropConstant.class, name = "wbpropconstant"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbQuantityExpr.class, name="wbquantityexpr"),
})
@Type(value = WbQuantityExpr.class, name = "wbquantityexpr"), })
public interface WbExpression<T> {
/**
* Evaluates the value expression in a given context,
* returns a Wikibase value suitable to be the target of a claim.
* Evaluates the value expression in a given context, returns a Wikibase value
* suitable to be the target of a claim.
*/
public T evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException;
public T evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException;
}

35
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -8,8 +31,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Represents an item that does not vary,
* it is independent of the row.
* Represents an item that does not vary, it is independent of the row.
*/
public class WbItemConstant implements WbExpression<ItemIdValue> {
@ -17,9 +39,7 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
private String label;
@JsonCreator
public WbItemConstant(
@JsonProperty("qid") String qid,
@JsonProperty("label") String label) {
public WbItemConstant(@JsonProperty("qid") String qid, @JsonProperty("label") String label) {
Validate.notNull(qid);
this.qid = qid;
Validate.notNull(label);
@ -28,10 +48,7 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
@Override
public ItemIdValue evaluate(ExpressionContext ctxt) {
return new SuggestedItemIdValue(
qid,
ctxt.getBaseIRI(),
label);
return new SuggestedItemIdValue(qid, ctxt.getBaseIRI(), label);
}
@JsonProperty("qid")

38
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.Collections;
@ -17,8 +40,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The representation of an item document, which can contain
* variables both for its own id and in its contents.
* The representation of an item document, which can contain variables both for
* its own id and in its contents.
*
* @author Antonin Delpeuch
*
@ -32,8 +55,7 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio
private List<WbStatementGroupExpr> statementGroups;
@JsonCreator
public WbItemDocumentExpr(
@JsonProperty("subject") WbExpression<? extends ItemIdValue> subjectExpr,
public WbItemDocumentExpr(@JsonProperty("subject") WbExpression<? extends ItemIdValue> subjectExpr,
@JsonProperty("nameDescs") List<WbNameDescExpr> nameDescExprs,
@JsonProperty("statementGroups") List<WbStatementGroupExpr> statementGroupExprs) {
Validate.notNull(subjectExpr);
@ -49,7 +71,8 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio
}
@Override
public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public ItemUpdate evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
ItemIdValue subjectId = getSubject().evaluate(ctxt);
ItemUpdateBuilder update = new ItemUpdateBuilder(subjectId);
for (WbStatementGroupExpr expr : getStatementGroups()) {
@ -88,9 +111,8 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio
return false;
}
WbItemDocumentExpr otherExpr = (WbItemDocumentExpr) other;
return subject.equals(otherExpr.getSubject()) &&
nameDescs.equals(otherExpr.getNameDescs()) &&
statementGroups.equals(otherExpr.getStatementGroups());
return subject.equals(otherExpr.getSubject()) && nameDescs.equals(otherExpr.getNameDescs())
&& statementGroups.equals(otherExpr.getStatementGroups());
}
@Override

34
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -1,6 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
@ -24,8 +46,8 @@ public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
@ -35,10 +57,10 @@ public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
}
@Override
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException {
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (cell.recon != null
&& (Judgment.Matched.equals(cell.recon.judgment) ||
Judgment.New.equals(cell.recon.judgment))) {
&& (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) {
return new ReconItemIdValue(cell.recon, cell.value.toString());
}
throw new SkipSchemaExpressionException();

37
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -19,9 +42,7 @@ public class WbLanguageConstant implements WbExpression<String> {
protected String _langLabel;
@JsonCreator
public WbLanguageConstant(
@JsonProperty("id") String langId,
@JsonProperty("label") String langLabel) {
public WbLanguageConstant(@JsonProperty("id") String langId, @JsonProperty("label") String langLabel) {
_langId = normalizeLanguageCode(langId);
Validate.notNull(_langId, "A valid language code must be provided.");
Validate.notNull(langLabel);
@ -29,13 +50,12 @@ public class WbLanguageConstant implements WbExpression<String> {
}
/**
* Checks that a language code is valid and returns its preferred
* version (converting deprecated language codes to their better values).
* Checks that a language code is valid and returns its preferred version
* (converting deprecated language codes to their better values).
*
* @param lang
* a Wikimedia language code
* @return
* the normalized code, or null if the code is invalid.
* @return the normalized code, or null if the code is invalid.
*/
public static String normalizeLanguageCode(String lang) {
try {
@ -47,7 +67,8 @@ public class WbLanguageConstant implements WbExpression<String> {
}
@Override
public String evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public String evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return _langId;
}

33
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -7,9 +30,9 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A language variable generates a language code from a cell.
* It checks its values against a known list of valid language codes
* and fixes on the fly the deprecated ones (see {@link WbLanguageConstant}).
* A language variable generates a language code from a cell. It checks its
* values against a known list of valid language codes and fixes on the fly the
* deprecated ones (see {@link WbLanguageConstant}).
*/
public class WbLanguageVariable extends WbVariableExpr<String> {
@ -18,8 +41,8 @@ public class WbLanguageVariable extends WbVariableExpr<String> {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from

38
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -11,7 +34,8 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant for a geographical location. The accepted format is lat,lng or lat/lng.
* A constant for a geographical location. The accepted format is lat,lng or
* lat/lng.
*
* @author Antonin Delpeuch
*
@ -24,8 +48,7 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
private GlobeCoordinatesValue parsed;
@JsonCreator
public WbLocationConstant(
@JsonProperty("value") String origValue) throws ParseException {
public WbLocationConstant(@JsonProperty("value") String origValue) throws ParseException {
this.value = origValue;
Validate.notNull(origValue);
this.parsed = parse(origValue);
@ -37,11 +60,11 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
*
* @param expr
* the string to parse
* @return
* the parsed location
* @return the parsed location
* @throws ParseException
*/
public static GlobeCoordinatesValue parse(String expr) throws ParseException {
public static GlobeCoordinatesValue parse(String expr)
throws ParseException {
double lat = 0;
double lng = 0;
double precision = defaultPrecision;
@ -53,8 +76,7 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
if (parts.length == 3) {
precision = Double.parseDouble(parts[2]);
}
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision,
GlobeCoordinatesValue.GLOBE_EARTH);
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision, GlobeCoordinatesValue.GLOBE_EARTH);
} catch (NumberFormatException e) {
;
}

24
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
@ -9,7 +32,6 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
@JsonCreator

36
extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -10,15 +33,13 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
private WbExpression<? extends String> languageExpr;
private WbExpression<? extends StringValue> valueExpr;
@JsonCreator
public WbMonolingualExpr(
@JsonProperty("language") WbExpression<? extends String> languageExpr,
public WbMonolingualExpr(@JsonProperty("language") WbExpression<? extends String> languageExpr,
@JsonProperty("value") WbExpression<? extends StringValue> valueExpr) {
Validate.notNull(languageExpr);
this.languageExpr = languageExpr;
@ -35,11 +56,7 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
return Datamodel.makeMonolingualTextValue(text, lang);
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"monolingual-text-without-language",
null,
QAWarning.Severity.WARNING,
1);
QAWarning warning = new QAWarning("monolingual-text-without-language", null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_text", text);
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
@ -62,8 +79,7 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
return false;
}
WbMonolingualExpr otherExpr = (WbMonolingualExpr) other;
return languageExpr.equals(otherExpr.getLanguageExpr()) &&
valueExpr.equals(otherExpr.getValueExpr());
return languageExpr.equals(otherExpr.getLanguageExpr()) && valueExpr.equals(otherExpr.getValueExpr());
}
@Override

39
extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -10,9 +33,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* An expression that represent a term (label, description or alias).
* The structure is slightly different from other expressions because
* we need to call different methods on {@link ItemUpdateBuilder}.
* An expression that represent a term (label, description or alias). The
* structure is slightly different from other expressions because we need to
* call different methods on {@link ItemUpdateBuilder}.
*
* @author Antonin Delpeuch
*
@ -21,17 +44,14 @@ import com.fasterxml.jackson.annotation.JsonProperty;
public class WbNameDescExpr {
enum NameDescrType {
LABEL,
DESCRIPTION,
ALIAS,
LABEL, DESCRIPTION, ALIAS,
}
private NameDescrType type;
private WbMonolingualExpr value;
@JsonCreator
public WbNameDescExpr(
@JsonProperty("name_type") NameDescrType type,
public WbNameDescExpr(@JsonProperty("name_type") NameDescrType type,
@JsonProperty("value") WbMonolingualExpr value) {
Validate.notNull(type);
this.type = type;
@ -82,8 +102,7 @@ public class WbNameDescExpr {
return false;
}
WbNameDescExpr otherExpr = (WbNameDescExpr) other;
return type.equals(otherExpr.getType()) &&
value.equals(otherExpr.getValue());
return type.equals(otherExpr.getType()) && value.equals(otherExpr.getValue());
}
@Override

30
extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -20,9 +43,7 @@ public class WbPropConstant implements WbExpression<PropertyIdValue> {
private String datatype;
@JsonCreator
public WbPropConstant(
@JsonProperty("pid") String pid,
@JsonProperty("label") String label,
public WbPropConstant(@JsonProperty("pid") String pid, @JsonProperty("label") String label,
@JsonProperty("datatype") String datatype) {
Validate.notNull(pid);
this.pid = pid;
@ -57,7 +78,8 @@ public class WbPropConstant implements WbExpression<PropertyIdValue> {
return false;
}
WbPropConstant otherConstant = (WbPropConstant) other;
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel()) && datatype.equals(otherConstant.getDatatype());
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel())
&& datatype.equals(otherConstant.getDatatype());
}
@Override

38
extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.math.BigDecimal;
@ -18,17 +41,16 @@ public class WbQuantityExpr implements WbExpression<QuantityValue> {
private final WbExpression<? extends ItemIdValue> unitExpr;
/**
* Creates an expression for a quantity, which
* contains two sub-expressions: one for the amount (a string with
* a particular format) and one for the unit, which is optional.
* Creates an expression for a quantity, which contains two sub-expressions: one
* for the amount (a string with a particular format) and one for the unit,
* which is optional.
*
* Setting unitExpr to null will give quantities without units. Setting
* it to a non-null value will make the unit mandatory: if the unit expression
* fails to evaluate, the whole quantity expression will fail too.
* Setting unitExpr to null will give quantities without units. Setting it to a
* non-null value will make the unit mandatory: if the unit expression fails to
* evaluate, the whole quantity expression will fail too.
*/
@JsonCreator
public WbQuantityExpr(
@JsonProperty("amount") WbExpression<? extends StringValue> amountExpr,
public WbQuantityExpr(@JsonProperty("amount") WbExpression<? extends StringValue> amountExpr,
@JsonProperty("unit") WbExpression<? extends ItemIdValue> unitExpr) {
Validate.notNull(amountExpr);
this.amountExpr = amountExpr;

30
extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -24,17 +47,18 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbReferenceExpr implements WbExpression<Reference> {
private List<WbSnakExpr> snakExprs;
@JsonCreator
public WbReferenceExpr(
@JsonProperty("snaks") List<WbSnakExpr> snakExprs) {
public WbReferenceExpr(@JsonProperty("snaks") List<WbSnakExpr> snakExprs) {
Validate.notNull(snakExprs);
this.snakExprs = snakExprs;
}
@Override
public Reference evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public Reference evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (WbSnakExpr expr : getSnaks()) {
List<Snak> snakList = new ArrayList<Snak>(1);

29
extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
@ -26,8 +49,7 @@ public class WbSnakExpr implements WbExpression<Snak> {
private WbExpression<? extends Value> value;
@JsonCreator
public WbSnakExpr(
@JsonProperty("prop") WbExpression<? extends PropertyIdValue> propExpr,
public WbSnakExpr(@JsonProperty("prop") WbExpression<? extends PropertyIdValue> propExpr,
@JsonProperty("value") WbExpression<? extends Value> valueExpr) {
Validate.notNull(propExpr);
this.prop = propExpr;
@ -36,7 +58,8 @@ public class WbSnakExpr implements WbExpression<Snak> {
}
@Override
public Snak evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
public Snak evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = getProp().evaluate(ctxt);
Value evaluatedValue = value.evaluate(ctxt);
return Datamodel.makeValueSnak(propertyId, evaluatedValue);

45
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -10,10 +33,10 @@ import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.Value;
@ -30,8 +53,7 @@ public class WbStatementExpr {
private List<WbReferenceExpr> referenceExprs;
@JsonCreator
public WbStatementExpr(
@JsonProperty("value") WbExpression<? extends Value> mainSnakValueExpr,
public WbStatementExpr(@JsonProperty("value") WbExpression<? extends Value> mainSnakValueExpr,
@JsonProperty("qualifiers") List<WbSnakExpr> qualifierExprs,
@JsonProperty("references") List<WbReferenceExpr> referenceExprs) {
Validate.notNull(mainSnakValueExpr);
@ -67,11 +89,7 @@ public class WbStatementExpr {
try {
qualifiers.add(qExpr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"ignored-qualifiers",
null,
QAWarning.Severity.INFO,
1);
QAWarning warning = new QAWarning("ignored-qualifiers", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
@ -86,11 +104,7 @@ public class WbStatementExpr {
try {
references.add(rExpr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"ignored-references",
null,
QAWarning.Severity.INFO,
1);
QAWarning warning = new QAWarning("ignored-references", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
@ -122,9 +136,8 @@ public class WbStatementExpr {
return false;
}
WbStatementExpr otherExpr = (WbStatementExpr) other;
return mainSnakValueExpr.equals(otherExpr.getMainsnak()) &&
qualifierExprs.equals(otherExpr.getQualifiers()) &&
referenceExprs.equals(otherExpr.getReferences());
return mainSnakValueExpr.equals(otherExpr.getMainsnak()) && qualifierExprs.equals(otherExpr.getQualifiers())
&& referenceExprs.equals(otherExpr.getReferences());
}
@Override

33
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -14,7 +37,6 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbStatementGroupExpr {
@ -23,8 +45,7 @@ public class WbStatementGroupExpr {
private List<WbStatementExpr> statementExprs;
@JsonCreator
public WbStatementGroupExpr(
@JsonProperty("property") WbExpression<? extends PropertyIdValue> propertyExpr,
public WbStatementGroupExpr(@JsonProperty("property") WbExpression<? extends PropertyIdValue> propertyExpr,
@JsonProperty("statements") List<WbStatementExpr> claimExprs) {
Validate.notNull(propertyExpr);
this.propertyExpr = propertyExpr;
@ -33,7 +54,8 @@ public class WbStatementGroupExpr {
this.statementExprs = claimExprs;
}
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) throws SkipSchemaExpressionException {
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = propertyExpr.evaluate(ctxt);
List<Statement> statements = new ArrayList<Statement>(statementExprs.size());
for (WbStatementExpr expr : statementExprs) {
@ -66,8 +88,7 @@ public class WbStatementGroupExpr {
return false;
}
WbStatementGroupExpr otherExpr = (WbStatementGroupExpr) other;
return propertyExpr.equals(otherExpr.getProperty()) &&
statementExprs.equals(otherExpr.getStatements());
return propertyExpr.equals(otherExpr.getProperty()) && statementExprs.equals(otherExpr.getStatements());
}
@Override

24
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
@ -7,7 +30,6 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbStringConstant implements WbExpression<StringValue> {
private String value;

27
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringVariable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -21,8 +44,8 @@ public class WbStringVariable extends WbVariableExpr<StringValue> {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from

51
extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -8,10 +31,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
/**
* A base class for expressions which draw their values
* from a particular column.
* A base class for expressions which draw their values from a particular
* column.
*
* @author antonin
* @author Antonin Delpeuch
*
* @param <T>
* the type of Wikibase value returned by the expression.
@ -30,8 +53,8 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
/**
* Returns the column name used by the variable.
* @return
* the OpenRefine column name
*
* @return the OpenRefine column name
*/
@JsonProperty("columnName")
public String getColumnName() {
@ -39,9 +62,8 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
}
/**
* Changes the column name used by the variable.
* This is useful for deserialization, as well as updates when
* column names change.
* Changes the column name used by the variable. This is useful for
* deserialization, as well as updates when column names change.
*/
@JsonProperty("columnName")
public void setColumnName(String columnName) {
@ -62,19 +84,18 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
}
/**
* Method that should be implemented by subclasses,
* converting an OpenRefine cell to a Wikibase value.
* Access to other values and emiting warnings is possible via
* the supplied EvaluationContext object.
* Method that should be implemented by subclasses, converting an OpenRefine
* cell to a Wikibase value. Access to other values and emiting warnings is
* possible via the supplied EvaluationContext object.
*
* @param cell
* the cell to convert
* @param ctxt
* the evaluation context
* @return
* the corresponding Wikibase value
* @return the corresponding Wikibase value
*/
public abstract T fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException;
public abstract T fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException;
/**
* Helper for equality methods of subclasses.

77
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
@ -8,6 +31,10 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -17,17 +44,10 @@ import com.google.refine.browsing.RowVisitor;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.openrefine.wikidata.schema.WbItemDocumentExpr;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.ExpressionContext;
import org.openrefine.wikidata.utils.JacksonJsonizable;
/**
* Main class representing a skeleton of Wikibase edits with
* OpenRefine columns as variables.
* Main class representing a skeleton of Wikibase edits with OpenRefine columns
* as variables.
*
* @author Antonin Delpeuch
*
@ -66,9 +86,9 @@ public class WikibaseSchema implements OverlayModel {
}
/**
* Evaluates all item documents in a particular expression context.
* This specifies, among others, a row where the values of the variables
* will be read.
* Evaluates all item documents in a particular expression context. This
* specifies, among others, a row where the values of the variables will be
* read.
*
* @param ctxt
* the context in which the schema should be evaluated.
@ -88,13 +108,13 @@ public class WikibaseSchema implements OverlayModel {
}
/**
* Evaluates the schema on a project, returning a list of ItemUpdates
* generated by the schema.
* Evaluates the schema on a project, returning a list of ItemUpdates generated
* by the schema.
*
* Some warnings will be emitted in the warning store: those are only
* the ones that are generated at evaluation time (such as invalid formats
* for dates). Issues detected on candidate statements (such as constraint
* violations) are not included at this stage.
* Some warnings will be emitted in the warning store: those are only the ones
* that are generated at evaluation time (such as invalid formats for dates).
* Issues detected on candidate statements (such as constraint violations) are
* not included at this stage.
*
* @param project
* the project on which the schema should be evaluated
@ -102,8 +122,7 @@ public class WikibaseSchema implements OverlayModel {
* the engine, which gives access to the current facets
* @param warningStore
* a store in which issues will be emitted
* @return item updates are stored in their
* generating order (not merged yet).
* @return item updates are stored in their generating order (not merged yet).
*/
public List<ItemUpdate> evaluate(Project project, Engine engine, QAWarningStore warningStore) {
List<ItemUpdate> result = new ArrayList<>();
@ -120,8 +139,10 @@ public class WikibaseSchema implements OverlayModel {
}
protected class EvaluatingRowVisitor implements RowVisitor {
private List<ItemUpdate> result;
private QAWarningStore warningStore;
public EvaluatingRowVisitor(List<ItemUpdate> result, QAWarningStore warningStore) {
this.result = result;
this.warningStore = warningStore;
@ -134,12 +155,7 @@ public class WikibaseSchema implements OverlayModel {
@Override
public boolean visit(Project project, int rowIndex, Row row) {
ExpressionContext ctxt = new ExpressionContext(
baseIri,
rowIndex,
row,
project.columnModel,
warningStore);
ExpressionContext ctxt = new ExpressionContext(baseIri, rowIndex, row, project.columnModel, warningStore);
result.addAll(evaluateItemDocuments(ctxt));
return false;
}
@ -150,12 +166,14 @@ public class WikibaseSchema implements OverlayModel {
}
}
static public WikibaseSchema reconstruct(JSONObject o) throws JSONException {
static public WikibaseSchema reconstruct(JSONObject o)
throws JSONException {
JSONArray changeArr = o.getJSONArray("itemDocuments");
WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) {
WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i), WbItemDocumentExpr.class);
WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i),
WbItemDocumentExpr.class);
schema.itemDocumentExprs.add(changeExpr);
}
return schema;
@ -174,7 +192,8 @@ public class WikibaseSchema implements OverlayModel {
writer.endObject();
}
static public WikibaseSchema load(Project project, JSONObject obj) throws Exception {
static public WikibaseSchema load(Project project, JSONObject obj)
throws Exception {
return reconstruct(obj);
}

50
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/PrefetchedEntityIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.List;
@ -5,34 +28,31 @@ import java.util.List;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
/**
* An entity id value that also comes with
* a label and possibly types.
* An entity id value that also comes with a label and possibly types.
*
* The rationale behind this classes is that OpenRefine
* already stores labels and types for the Wikidata items
* it knows about (in the reconciliation data), so it is
* worth keeping this data to avoid re-fetching it when
* we need it.
* The rationale behind this classes is that OpenRefine already stores labels
* and types for the Wikidata items it knows about (in the reconciliation data),
* so it is worth keeping this data to avoid re-fetching it when we need it.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public interface PrefetchedEntityIdValue extends EntityIdValue {
/**
* This should return the label "as we got it", with no guarantee
* that it is current or that its language matches that of the user.
* In general though, that should be the case if the user always uses
* OpenRefine with the same language settings.
* This should return the label "as we got it", with no guarantee that it is
* current or that its language matches that of the user. In general though,
* that should be the case if the user always uses OpenRefine with the same
* language settings.
*
* @return the preferred label of the entity
*/
public String getLabel();
/**
* Returns a list of types for this item. Again these are the types
* as they were originally fetched from the reconciliation interface:
* they can diverge from what is currently on the item.
* Returns a list of types for this item. Again these are the types as they were
* originally fetched from the reconciliation interface: they can diverge from
* what is currently on the item.
*
* Empty lists should be returned for
*/

55
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.ArrayList;
@ -14,19 +37,18 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.refine.model.Recon;
/**
* An EntityIdValue that holds not just the id but also
* the label as fetched by either the reconciliation interface
* or the suggester and its type, both stored as reconciliation
* candidates.
* An EntityIdValue that holds not just the id but also the label as fetched by
* either the reconciliation interface or the suggester and its type, both
* stored as reconciliation candidates.
*
* This label will be localized depending on the language chosen
* by the user for OpenRefine's interface. Storing it lets us
* reuse it later on without having to re-fetch it.
* This label will be localized depending on the language chosen by the user for
* OpenRefine's interface. Storing it lets us reuse it later on without having
* to re-fetch it.
*
* Storing the types also lets us perform some constraint checks
* without re-fetching the types of many items.
* Storing the types also lets us perform some constraint checks without
* re-fetching the types of many items.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
@ -37,8 +59,7 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
public ReconEntityIdValue(Recon match, String cellValue) {
_recon = match;
_cellValue = cellValue;
assert (Recon.Judgment.Matched.equals(_recon.judgment) ||
Recon.Judgment.New.equals(_recon.judgment));
assert (Recon.Judgment.Matched.equals(_recon.judgment) || Recon.Judgment.New.equals(_recon.judgment));
}
@JsonIgnore
@ -71,22 +92,18 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue {
public abstract String getEntityType();
/**
* Returns the integer used internally in OpenRefine to identify the new
* item.
* Returns the integer used internally in OpenRefine to identify the new item.
*
* @return
* the judgment history entry id of the reconciled cell
* @return the judgment history entry id of the reconciled cell
*/
public long getReconInternalId() {
return getRecon().judgmentHistoryEntry;
}
/**
* Returns the reconciliation object corresponding to this entity.
*
* @return
* the full reconciliation metadata of the corresponding cell
* @return the full reconciliation metadata of the corresponding cell
*/
public Recon getRecon() {
return _recon;

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconItemIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconPropertyIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;

33
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java
View File

@ -1,18 +1,40 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.ArrayList;
import java.util.List;
import org.wikidata.wdtk.datamodel.helpers.Hash;
import org.wikidata.wdtk.datamodel.helpers.ToString;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* An EntityIdValue that we have obtained from a suggest widget
* in the schema alignment dialog.
* An EntityIdValue that we have obtained from a suggest widget in the schema
* alignment dialog.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue {
@ -59,8 +81,7 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
@Override
public boolean equals(Object other) {
if (other == null ||
!EntityIdValue.class.isInstance(other)) {
if (other == null || !EntityIdValue.class.isInstance(other)) {
return false;
}
final EntityIdValue otherNew = (EntityIdValue) other;

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedItemIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;

23
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.helpers.ToString;

25
extensions/wikidata/src/org/openrefine/wikidata/schema/exceptions/InvalidSchemaException.java
View File

@ -1,6 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.exceptions;
public class InvalidSchemaException extends Exception {
static final long serialVersionUID = 494837587034L;
}

25
extensions/wikidata/src/org/openrefine/wikidata/schema/exceptions/SkipSchemaExpressionException.java
View File

@ -1,6 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.exceptions;
public class SkipSchemaExpressionException extends Exception {
static final long serialVersionUID = 738592057L;
}

90
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates;
import java.util.ArrayList;
@ -23,13 +46,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to plan an update of an item, after evaluating the statements
* but before fetching the current content of the item (this is why it does not
* A class to plan an update of an item, after evaluating the statements but
* before fetching the current content of the item (this is why it does not
* extend StatementsUpdate).
*
* @author Antonin Delpeuch
*/
public class ItemUpdate {
private final ItemIdValue qid;
private final List<Statement> addedStatements;
private final Set<Statement> deletedStatements;
@ -41,7 +65,8 @@ public class ItemUpdate {
* Constructor.
*
* @param qid
* the subject of the document. It can be a reconciled item value for new items.
* the subject of the document. It can be a reconciled item value for
* new items.
* @param addedStatements
* the statements to add on the item. They should be distinct. They
* are modelled as a list because their insertion order matters.
@ -52,13 +77,12 @@ public class ItemUpdate {
* @param descriptions
* the descriptions to add on the item
* @param aliases
* the aliases to add on the item. In theory their order should matter
* but in practice people rarely rely on the order of aliases so this
* is just kept as a set for simplicity.
* the aliases to add on the item. In theory their order should
* matter but in practice people rarely rely on the order of aliases
* so this is just kept as a set for simplicity.
*/
@JsonCreator
public ItemUpdate(
@JsonProperty("subject") ItemIdValue qid,
public ItemUpdate(@JsonProperty("subject") ItemIdValue qid,
@JsonProperty("addedStatements") List<Statement> addedStatements,
@JsonProperty("deletedStatements") Set<Statement> deletedStatements,
@JsonProperty("labels") Set<MonolingualTextValue> labels,
@ -97,8 +121,8 @@ public class ItemUpdate {
}
/**
* Added statements are recorded as a list because
* their order of insertion matters.
* Added statements are recorded as a list because their order of insertion
* matters.
*
* @return the list of all added statements
*/
@ -152,16 +176,13 @@ public class ItemUpdate {
*/
@JsonIgnore
public boolean isEmpty() {
return (addedStatements.isEmpty()
&& deletedStatements.isEmpty()
&& labels.isEmpty()
&& descriptions.isEmpty()
return (addedStatements.isEmpty() && deletedStatements.isEmpty() && labels.isEmpty() && descriptions.isEmpty()
&& aliases.isEmpty());
}
/**
* Merges all the changes in other into this instance.
* Both updates should have the same subject.
* Merges all the changes in other into this instance. Both updates should have
* the same subject.
*
* @param other
* the other change that should be merged
@ -182,14 +203,11 @@ public class ItemUpdate {
newDescriptions.addAll(other.getDescriptions());
Set<MonolingualTextValue> newAliases = new HashSet<>(aliases);
newAliases.addAll(other.getAliases());
return new ItemUpdate(
qid, newAddedStatements, newDeletedStatements,
newLabels, newDescriptions, newAliases);
return new ItemUpdate(qid, newAddedStatements, newDeletedStatements, newLabels, newDescriptions, newAliases);
}
/**
* Group added statements in StatementGroups: useful if the
* item is new.
* Group added statements in StatementGroups: useful if the item is new.
*
* @return a grouped version of getAddedStatements()
*/
@ -210,8 +228,8 @@ public class ItemUpdate {
}
/**
* Group a list of ItemUpdates by subject: this is useful to make one single edit
* per item.
* Group a list of ItemUpdates by subject: this is useful to make one single
* edit per item.
*
* @param itemDocuments
* @return a map from item ids to merged ItemUpdate for that id
@ -242,15 +260,12 @@ public class ItemUpdate {
}
/**
* This should only be used when creating a new item.
* This ensures that we never add an alias without adding
* a label in the same language.
* This should only be used when creating a new item. This ensures that we never
* add an alias without adding a label in the same language.
*/
public ItemUpdate normalizeLabelsAndAliases() {
// Ensure that we are only adding aliases with labels
Set<String> labelLanguages = labels.stream()
.map(l -> l.getLanguageCode())
.collect(Collectors.toSet());
Set<String> labelLanguages = labels.stream().map(l -> l.getLanguageCode()).collect(Collectors.toSet());
Set<MonolingualTextValue> filteredAliases = new HashSet<>();
Set<MonolingualTextValue> newLabels = new HashSet<>(labels);
@ -262,8 +277,7 @@ public class ItemUpdate {
filteredAliases.add(alias);
}
}
return new ItemUpdate(qid, addedStatements, deletedStatements,
newLabels, descriptions, filteredAliases);
return new ItemUpdate(qid, addedStatements, deletedStatements, newLabels, descriptions, filteredAliases);
}
@Override
@ -272,18 +286,16 @@ public class ItemUpdate {
return false;
}
ItemUpdate otherUpdate = (ItemUpdate) other;
return qid.equals(otherUpdate.getItemId())&&
addedStatements.equals(otherUpdate.getAddedStatements()) &&
deletedStatements.equals(otherUpdate.getDeletedStatements()) &&
labels.equals(otherUpdate.getLabels()) &&
descriptions.equals(otherUpdate.getDescriptions()) &&
aliases.equals(otherUpdate.getAliases());
return qid.equals(otherUpdate.getItemId()) && addedStatements.equals(otherUpdate.getAddedStatements())
&& deletedStatements.equals(otherUpdate.getDeletedStatements())
&& labels.equals(otherUpdate.getLabels()) && descriptions.equals(otherUpdate.getDescriptions())
&& aliases.equals(otherUpdate.getAliases());
}
@Override
public int hashCode() {
return qid.hashCode() + addedStatements.hashCode() + deletedStatements.hashCode() +
labels.hashCode() + descriptions.hashCode() + aliases.hashCode();
return qid.hashCode() + addedStatements.hashCode() + deletedStatements.hashCode() + labels.hashCode()
+ descriptions.hashCode() + aliases.hashCode();
}
@Override

64
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates;
import java.util.Set;
@ -10,7 +33,6 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* Constructs a {@link ItemUpdate} incrementally.
*
@ -18,6 +40,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
*
*/
public class ItemUpdateBuilder {
private ItemIdValue qid;
private List<Statement> addedStatements;
private Set<Statement> deletedStatements;
@ -30,7 +53,8 @@ public class ItemUpdateBuilder {
* Constructor.
*
* @param qid
* the subject of the document. It can be a reconciled item value for new items.
* the subject of the document. It can be a reconciled item value for
* new items.
*/
public ItemUpdateBuilder(ItemIdValue qid) {
Validate.notNull(qid);
@ -44,8 +68,8 @@ public class ItemUpdateBuilder {
}
/**
* Mark a statement for insertion. If it matches an existing
* statement, it will update the statement instead.
* Mark a statement for insertion. If it matches an existing statement, it will
* update the statement instead.
*
* @param statement
* the statement to add or update
@ -57,8 +81,8 @@ public class ItemUpdateBuilder {
}
/**
* Mark a statement for deletion. If no such statement exists,
* nothing will be deleted.
* Mark a statement for deletion. If no such statement exists, nothing will be
* deleted.
*
* @param statement
* the statement to delete
@ -94,8 +118,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds a label to the item. It will override any
* existing label in this language.
* Adds a label to the item. It will override any existing label in this
* language.
*
* @param label
* the label to add
@ -107,8 +131,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds a list of labels to the item. It will override any
* existing label in each language.
* Adds a list of labels to the item. It will override any existing label in
* each language.
*
* @param labels
* the labels to add
@ -120,8 +144,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds a description to the item. It will override any existing
* description in this language.
* Adds a description to the item. It will override any existing description in
* this language.
*
* @param description
* the description to add
@ -133,8 +157,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds a list of descriptions to the item. It will override any
* existing description in each language.
* Adds a list of descriptions to the item. It will override any existing
* description in each language.
*
* @param descriptions
* the descriptions to add
@ -146,8 +170,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds an alias to the item. It will be added to any existing
* aliases in that language.
* Adds an alias to the item. It will be added to any existing aliases in that
* language.
*
* @param alias
* the alias to add
@ -159,8 +183,8 @@ public class ItemUpdateBuilder {
}
/**
* Adds a list of aliases to the item. They will be added to any
* existing aliases in each language.
* Adds a list of aliases to the item. They will be added to any existing
* aliases in each language.
*
* @param aliases
* the aliases to add
@ -173,12 +197,12 @@ public class ItemUpdateBuilder {
/**
* Constructs the {@link ItemUpdate}.
*
* @return
*/
public ItemUpdate build() {
built = true;
return new ItemUpdate(qid, addedStatements, deletedStatements,
labels, descriptions, aliases);
return new ItemUpdate(qid, addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

24
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/ImpossibleSchedulingException.java
View File

@ -1,6 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
public class ImpossibleSchedulingException extends Exception {
private static final long serialVersionUID = 6621563898380564148L;

52
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.Collections;
@ -5,7 +28,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -21,8 +43,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* A class that extracts the new entity ids referred to
* in a statement.
* A class that extracts the new entity ids referred to in a statement.
*
* @author Antonin Delpeuch
*
@ -30,21 +51,18 @@ import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
/**
* Extracts all the new entities mentioned by this statement. This
* does not include the subject of the statement.
* Extracts all the new entities mentioned by this statement. This does not
* include the subject of the statement.
*
* @param statement
* the statement to inspect
* @return
* the set of all new entities mentioned by the statement
* @return the set of all new entities mentioned by the statement
*/
public Set<ReconItemIdValue> extractPointers(Statement statement) {
Set<ReconItemIdValue> result = new HashSet<>();
result.addAll(extractPointers(statement.getClaim().getMainSnak()));
result.addAll(extractPointers(statement.getClaim().getQualifiers()));
statement.getReferences().stream()
.map(l -> extractPointers(l.getSnakGroups()))
.forEach(s -> result.addAll(s));
statement.getReferences().stream().map(l -> extractPointers(l.getSnakGroups())).forEach(s -> result.addAll(s));
return result;
}
@ -56,9 +74,7 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
*/
public Set<ReconItemIdValue> extractPointers(List<SnakGroup> snakGroups) {
Set<ReconItemIdValue> result = new HashSet<>();
snakGroups.stream()
.map(s -> extractPointers(s))
.forEach(s -> result.addAll(s));
snakGroups.stream().map(s -> extractPointers(s)).forEach(s -> result.addAll(s));
return result;
}
@ -70,16 +86,14 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
*/
public Set<ReconItemIdValue> extractPointers(SnakGroup snakGroup) {
Set<ReconItemIdValue> result = new HashSet<>();
snakGroup.getSnaks().stream()
.map(s -> extractPointers(s))
.forEach(s -> result.addAll(s));
snakGroup.getSnaks().stream().map(s -> extractPointers(s)).forEach(s -> result.addAll(s));
return result;
}
/**
* Extracts all new entities mentioned by this snak group.
* Currently there will be at most one: the target of the snak
* (as property ids cannot be new for now).
* Extracts all new entities mentioned by this snak group. Currently there will
* be at most one: the target of the snak (as property ids cannot be new for
* now).
*
* @param snak
* @return

50
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/QuickStatementsUpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -8,45 +31,41 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class QuickStatementsUpdateScheduler implements UpdateScheduler {
private PointerExtractor extractor = new PointerExtractor();
/**
* This map holds for each new entity id value a list of updates
* that refer to this id (and should hence be scheduled right after
* creation of that entity).
* This map holds for each new entity id value a list of updates that refer to
* this id (and should hence be scheduled right after creation of that entity).
*/
private Map<ItemIdValue, UpdateSequence> pointerUpdates;
/**
* This contains all updates which do not refer to any new entity
* apart from possibly the subject, in the order that they were supplied to us.
* This contains all updates which do not refer to any new entity apart from
* possibly the subject, in the order that they were supplied to us.
*/
private UpdateSequence pointerFreeUpdates;
/**
* Separates out the statements which refer to new items from the rest
* of the update. The resulting updates are stored in {@link referencingUpdates}
* and {@link updatesWithoutReferences}.
* Separates out the statements which refer to new items from the rest of the
* update. The resulting updates are stored in {@link referencingUpdates} and
* {@link updatesWithoutReferences}.
*
* @param update
* @throws ImpossibleSchedulingException
* if two new item ids are referred to in the same statement
*/
protected void splitUpdate(ItemUpdate update) throws ImpossibleSchedulingException {
protected void splitUpdate(ItemUpdate update)
throws ImpossibleSchedulingException {
ItemUpdateBuilder remainingUpdateBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels())
.addDescriptions(update.getDescriptions())
.addAliases(update.getAliases())
.addLabels(update.getLabels()).addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
Map<ItemIdValue, ItemUpdateBuilder> referencingUpdates = new HashMap<>();
@ -87,7 +106,8 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
}
@Override
public List<ItemUpdate> schedule(List<ItemUpdate> updates) throws ImpossibleSchedulingException {
public List<ItemUpdate> schedule(List<ItemUpdate> updates)
throws ImpossibleSchedulingException {
pointerUpdates = new HashMap<>();
pointerFreeUpdates = new UpdateSequence();

43
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/UpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.List;
@ -5,11 +28,9 @@ import java.util.List;
import org.openrefine.wikidata.updates.ItemUpdate;
/**
* A scheduling strategy for item updates.
* Given a list of initial updates, the scheduler
* reorganizes these updates (possibly splitting them
* or merging them) to create a sequence that is suitable
* for a particular import process.
* A scheduling strategy for item updates. Given a list of initial updates, the
* scheduler reorganizes these updates (possibly splitting them or merging them)
* to create a sequence that is suitable for a particular import process.
*
* @author Antonin Delpeuch
*
@ -17,16 +38,16 @@ import org.openrefine.wikidata.updates.ItemUpdate;
public interface UpdateScheduler {
/**
* Performs the scheduling. The initial updates are provided
* as a list so that the scheduler can attempt to respect the
* initial order (but no guarantee is made for that in general).
* Performs the scheduling. The initial updates are provided as a list so that
* the scheduler can attempt to respect the initial order (but no guarantee is
* made for that in general).
*
* @param updates
* the updates to schedule
* @return
* the reorganized updates
* @return the reorganized updates
* @throws ImpossibleSchedulingException
* when the scheduler cannot cope with a particular edit plan.
*/
public List<ItemUpdate> schedule(List<ItemUpdate> updates) throws ImpossibleSchedulingException;
public List<ItemUpdate> schedule(List<ItemUpdate> updates)
throws ImpossibleSchedulingException;
}

32
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/UpdateSequence.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -10,12 +33,13 @@ import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
/**
* Helper class to store a list of updates where each subject
* appears at most once. It preserves order of insertion.
* Helper class to store a list of updates where each subject appears at most
* once. It preserves order of insertion.
*
* @author Antonin Delpeuch
*/
public class UpdateSequence {
/**
* The list of updates stored by this container
*/
@ -26,8 +50,8 @@ public class UpdateSequence {
private Map<ItemIdValue, Integer> index = new HashMap<>();
/**
* Adds a new update to the list, merging it with any existing
* one with the same subject.
* Adds a new update to the list, merging it with any existing one with the same
* subject.
*
* @param update
*/

51
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/WikibaseAPIUpdateScheduler.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates.scheduler;
import java.util.ArrayList;
@ -15,11 +38,10 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A simple scheduler for batches commited via the Wikibase API.
*
* The strategy is quite simple and makes at most two edits
* per touched item (which is not minimal though). Each update
* is split between statements making references to new items,
* and statements not making these references. All updates with no
* references to new items are done first (which creates all new
* The strategy is quite simple and makes at most two edits per touched item
* (which is not minimal though). Each update is split between statements making
* references to new items, and statements not making these references. All
* updates with no references to new items are done first (which creates all new
* items), then all other updates are done.
*
* @author Antonin Delpeuch
@ -28,13 +50,13 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
/**
* The first part of updates: the ones which create new items
* without referring to any other new item.
* The first part of updates: the ones which create new items without referring
* to any other new item.
*/
private UpdateSequence pointerFreeUpdates;
/**
* The second part of the updates: all existing items, plus
* all parts of new items that refer to other new items.
* The second part of the updates: all existing items, plus all parts of new
* items that refer to other new items.
*/
private UpdateSequence pointerFullUpdates;
/**
@ -62,9 +84,7 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
Set<ItemIdValue> unseenPointers = new HashSet<>(allPointers);
unseenPointers.removeAll(pointerFreeUpdates.getSubjects());
result.addAll(unseenPointers.stream()
.map(e -> new ItemUpdateBuilder(e).build())
.collect(Collectors.toList()));
result.addAll(unseenPointers.stream().map(e -> new ItemUpdateBuilder(e).build()).collect(Collectors.toList()));
// Part 2: add all the pointer full updates
result.addAll(pointerFullUpdates.getUpdates());
@ -74,13 +94,12 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
/**
* Splits an update into two parts
*
* @param update
*/
protected void splitUpdate(ItemUpdate update) {
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels())
.addDescriptions(update.getDescriptions())
.addAliases(update.getAliases())
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId()).addLabels(update.getLabels())
.addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
ItemUpdateBuilder pointerFullBuilder = new ItemUpdateBuilder(update.getItemId());

35
extensions/wikidata/src/org/openrefine/wikidata/utils/EntityCache.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.util.concurrent.TimeUnit;
@ -14,6 +37,7 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
public class EntityCache {
private static EntityCache _entityCache = new EntityCache();
private LoadingCache<String, EntityDocument> _cache = null;
@ -23,12 +47,11 @@ public class EntityCache {
ApiConnection connection = ApiConnection.getWikidataApiConnection();
_fetcher = new WikibaseDataFetcher(connection, Datamodel.SITE_WIKIDATA);
_cache = CacheBuilder.newBuilder()
.maximumSize(4096)
.expireAfterWrite(1, TimeUnit.HOURS)
.build(
new CacheLoader<String, EntityDocument>() {
public EntityDocument load(String entityId) throws Exception {
_cache = CacheBuilder.newBuilder().maximumSize(4096).expireAfterWrite(1, TimeUnit.HOURS)
.build(new CacheLoader<String, EntityDocument>() {
public EntityDocument load(String entityId)
throws Exception {
EntityDocument doc = _fetcher.getEntityDocument(entityId);
if (doc != null) {
return doc;

31
extensions/wikidata/src/org/openrefine/wikidata/utils/FirstLinesExtractor.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.io.IOException;
@ -6,17 +29,19 @@ import java.io.StringReader;
import java.io.StringWriter;
public class FirstLinesExtractor {
/**
* Returns the first n lines of a given string
*
* @param content
* the content, where lines are separated by '\n'
* @param nbLines
* the number of lines to extract
* @return
* the first lines of the string
* @return the first lines of the string
* @throws IOException
*/
public static String extractFirstLines(String content, int nbLines) throws IOException {
public static String extractFirstLines(String content, int nbLines)
throws IOException {
StringWriter stringWriter = new StringWriter();
LineNumberReader reader = new LineNumberReader(new StringReader(content));

37
extensions/wikidata/src/org/openrefine/wikidata/utils/JacksonJsonizable.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.io.IOException;
@ -16,13 +39,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.Jsonizable;
/**
* This class is inefficient because it serializes the
* object to string and then deserializes it back. Unfortunately,
* this is the only simple way to bridge Jackson to org.json.
* This conversion should be removed when (if ?) we migrate OpenRefine
* a better JSON library.
* This class is inefficient because it serializes the object to string and then
* deserializes it back. Unfortunately, this is the only simple way to bridge
* Jackson to org.json. This conversion should be removed when (if ?) we migrate
* OpenRefine a better JSON library.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@ -39,7 +61,8 @@ public abstract class JacksonJsonizable implements Jsonizable {
}
}
public static <T> T fromJSONClass(JSONObject obj, Class<T> klass) throws JSONException {
public static <T> T fromJSONClass(JSONObject obj, Class<T> klass)
throws JSONException {
ObjectMapper mapper = new ObjectMapper();
String json = obj.toString();
try {

52
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java
View File

@ -1,33 +1,49 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
public abstract class CommandTest extends RefineTest {
protected Project project = null;
protected HttpServletRequest request = null;
protected HttpServletResponse response = null;
@ -36,7 +52,8 @@ public abstract class CommandTest extends RefineTest {
protected Command command = null;
@BeforeMethod(alwaysRun = true)
public void setUpProject() throws JSONException {
public void setUpProject()
throws JSONException {
project = createCSVProject(TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
request = mock(HttpServletRequest.class);
@ -53,5 +70,4 @@ public abstract class CommandTest extends RefineTest {
}
}
}

42
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java
View File

@ -1,5 +1,34 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
import java.io.IOException;
import javax.servlet.ServletException;
import org.json.JSONException;
@ -8,24 +37,19 @@ import org.openrefine.wikidata.testing.TestingData;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
import java.io.IOException;
import com.google.refine.util.ParsingUtilities;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest {
@BeforeMethod
public void SetUp() throws JSONException {
public void SetUp()
throws JSONException {
command = new PreviewWikibaseSchemaCommand();
}
@Test
public void testValidSchema() throws JSONException, IOException, ServletException {
public void testValidSchema()
throws JSONException, IOException, ServletException {
String schemaJson = jsonFromFile("data/schema/inception.json").toString();
when(request.getParameter("schema")).thenReturn(schemaJson);

35
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommandTest.java
View File

@ -1,16 +1,38 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
import java.io.IOException;
import javax.servlet.ServletException;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class SaveWikibaseSchemaCommandTest extends SchemaCommandTest {
@ -20,7 +42,8 @@ public class SaveWikibaseSchemaCommandTest extends SchemaCommandTest {
}
@Test
public void testValidSchema() throws ServletException, IOException {
public void testValidSchema()
throws ServletException, IOException {
String schemaJson = jsonFromFile("data/schema/inception.json").toString();
when(request.getParameter("schema")).thenReturn(schemaJson);

34
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java
View File

@ -1,8 +1,28 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
@ -10,17 +30,21 @@ import java.io.IOException;
import javax.servlet.ServletException;
import org.testng.annotations.Test;
public abstract class SchemaCommandTest extends CommandTest {
@Test
public void testNoSchema() throws ServletException, IOException {
public void testNoSchema()
throws ServletException, IOException {
command.doPost(request, response);
assertEquals("{\"status\":\"error\",\"message\":\"No Wikibase schema provided.\"}", writer.toString());
}
@Test
public void testInvalidSchema() throws ServletException, IOException {
public void testInvalidSchema()
throws ServletException, IOException {
when(request.getParameter("schema")).thenReturn("{bogus json");
command.doPost(request, response);

56
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import org.openrefine.wikidata.testing.TestingData;
@ -31,7 +54,6 @@ import java.util.stream.Collectors;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
public class EditBatchProcessorTest extends RefineTest {
private WikibaseDataFetcher fetcher = null;
@ -48,27 +70,23 @@ public class EditBatchProcessorTest extends RefineTest {
}
@Test
public void testNewItem() throws InterruptedException, MediaWikiApiErrorException, IOException {
public void testNewItem()
throws InterruptedException, MediaWikiApiErrorException, IOException {
List<ItemUpdate> batch = new ArrayList<>();
batch.add(new ItemUpdateBuilder(TestingData.existingId)
.addAlias(Datamodel.makeMonolingualTextValue("my new alias", "en"))
.addStatement(TestingData.generateStatement(TestingData.existingId, TestingData.newIdA))
.build());
.addStatement(TestingData.generateStatement(TestingData.existingId, TestingData.newIdA)).build());
MonolingualTextValue label = Datamodel.makeMonolingualTextValue("better label", "en");
batch.add(new ItemUpdateBuilder(TestingData.newIdA)
.addAlias(label)
.build());
batch.add(new ItemUpdateBuilder(TestingData.newIdA).addAlias(label).build());
// Plan expected edits
ItemDocument existingItem = ItemDocumentBuilder.forItemId(TestingData.existingId)
.withLabel(Datamodel.makeMonolingualTextValue("pomme", "fr"))
.withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr"))
.build();
.withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr")).build();
when(fetcher.getEntityDocuments(Collections.singletonList(TestingData.existingId.getId())))
.thenReturn(Collections.singletonMap(TestingData.existingId.getId(), existingItem));
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA)
.withLabel(label).build();
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234"))
.withLabel(label).withRevisionId(37828L).build();
when(editor.createItemDocument(expectedNewItem, summary)).thenReturn(createdNewItem);
@ -92,27 +110,24 @@ public class EditBatchProcessorTest extends RefineTest {
}
@Test
public void testMultipleBatches() throws MediaWikiApiErrorException, InterruptedException, IOException {
public void testMultipleBatches()
throws MediaWikiApiErrorException, InterruptedException, IOException {
// Prepare test data
MonolingualTextValue description = Datamodel.makeMonolingualTextValue("village in Nepal", "en");
List<String> ids = new ArrayList<>();
for (int i = 124; i < 190; i++) {
ids.add("Q" + String.valueOf(i));
}
List<ItemIdValue> qids = ids.stream()
.map(e -> Datamodel.makeWikidataItemIdValue(e))
List<ItemIdValue> qids = ids.stream().map(e -> Datamodel.makeWikidataItemIdValue(e))
.collect(Collectors.toList());
List<ItemUpdate> batch = qids.stream()
.map(qid -> new ItemUpdateBuilder(qid)
.addDescription(description)
.build())
.map(qid -> new ItemUpdateBuilder(qid).addDescription(description).build())
.collect(Collectors.toList());
int batchSize = 50;
List<ItemDocument> fullBatch = qids.stream()
.map(qid -> ItemDocumentBuilder.forItemId(qid)
.withStatement(TestingData.generateStatement(qid, TestingData.existingId))
.build())
.withStatement(TestingData.generateStatement(qid, TestingData.existingId)).build())
.collect(Collectors.toList());
List<ItemDocument> firstBatch = fullBatch.subList(0, batchSize);
List<ItemDocument> secondBatch = fullBatch.subList(batchSize, fullBatch.size());
@ -142,8 +157,7 @@ public class EditBatchProcessorTest extends RefineTest {
}
private Map<String, EntityDocument> toMap(List<ItemDocument> docs) {
return docs.stream()
.collect(Collectors.toMap(doc -> doc.getItemId().getId(), doc -> doc));
return docs.stream().collect(Collectors.toMap(doc -> doc.getItemId().getId(), doc -> doc));
}
private List<String> toQids(List<ItemDocument> docs) {

24
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import static org.junit.Assert.assertEquals;
@ -13,6 +36,7 @@ import com.google.refine.model.Recon;
import com.google.refine.tests.RefineTest;
public class NewItemLibraryTest extends RefineTest {
private NewItemLibrary library;
@BeforeMethod

29
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/ReconEntityRewriterTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import static org.junit.Assert.assertEquals;
@ -52,16 +75,14 @@ public class ReconEntityRewriterTest {
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de"))
.build();
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
ItemUpdate rewritten = rewriter.rewrite(update);
ItemUpdate expected = new ItemUpdateBuilder(subject)
.addStatement(TestingData.generateStatement(subject, newlyCreated))
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de"))
.build();
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
assertEquals(expected, rewritten);
}
}

49
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java
View File

@ -1,7 +1,29 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.math.BigDecimal;
@ -51,8 +73,10 @@ public class QSValuePrinterTest {
@Test
public void printGlobeCoordinate() {
// I don't see how to avoid the trailing zeros - in any case it's not a big deal because
// the precision is governed by a different parameter that QuickStatements does not support.
// I don't see how to avoid the trailing zeros - in any case it's not a big deal
// because
// the precision is governed by a different parameter that QuickStatements does
// not support.
assertPrints("@43.261930/10.927080", Datamodel.makeGlobeCoordinatesValue(43.26193, 10.92708,
GlobeCoordinatesValue.PREC_DEGREE, GlobeCoordinatesValue.GLOBE_EARTH));
}
@ -68,20 +92,19 @@ public class QSValuePrinterTest {
@Test
public void printSimpleQuantityValue() {
assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
null, null, "1"));
assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null, "1"));
}
@Test
public void printQuantityValueWithUnit() {
assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
null, null, "http://www.wikidata.org/entity/Q11573"));
assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null,
"http://www.wikidata.org/entity/Q11573"));
}
@Test
public void printQuantityValueWithBounds() {
assertPrints("10.00[9.0,11.05]", Datamodel.makeQuantityValue(new BigDecimal("10.00"),
new BigDecimal("9.0"), new BigDecimal("11.05"), "1"));
assertPrints("10.00[9.0,11.05]", Datamodel.makeQuantityValue(new BigDecimal("10.00"), new BigDecimal("9.0"),
new BigDecimal("11.05"), "1"));
}
@Test
@ -101,13 +124,13 @@ public class QSValuePrinterTest {
@Test
public void printYear() {
assertPrints("+1586-00-00T00:00:00Z/9", Datamodel.makeTimeValue(1586L, (byte)0, (byte)0, (byte)0,
(byte)0, (byte)0, (byte)9, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
assertPrints("+1586-00-00T00:00:00Z/9", Datamodel.makeTimeValue(1586L, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte) 0, (byte) 9, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
}
@Test
public void printDay() {
assertPrints("+1586-03-09T00:00:00Z/11", Datamodel.makeTimeValue(1586L, (byte)3, (byte)9, (byte)0,
(byte)0, (byte)0, (byte)11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
assertPrints("+1586-03-09T00:00:00Z/11", Datamodel.makeTimeValue(1586L, (byte) 3, (byte) 9, (byte) 0, (byte) 0,
(byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO));
}
}

68
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import static org.junit.Assert.assertEquals;
@ -14,7 +37,6 @@ import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.updates.scheduler.UpdateSchedulerTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
@ -36,16 +58,17 @@ public class QuickStatementsExporterTest extends RefineTest {
private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377");
private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528");
private String export(ItemUpdate... itemUpdates) throws IOException {
private String export(ItemUpdate... itemUpdates)
throws IOException {
StringWriter writer = new StringWriter();
exporter.translateItemList(Arrays.asList(itemUpdates), writer);
return writer.toString();
}
@Test
public void testSimpleProject() throws JSONException, IOException {
Project project = this.createCSVProject(
TestingData.inceptionWithNewCsv);
public void testSimpleProject()
throws JSONException, IOException {
Project project = this.createCSVProject(TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
JSONObject serialized = TestingData.jsonFromFile("data/schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
@ -59,40 +82,38 @@ public class QuickStatementsExporterTest extends RefineTest {
}
@Test
public void testImpossibleScheduling() throws IOException {
public void testImpossibleScheduling()
throws IOException {
Statement sNewAtoNewB = TestingData.generateStatement(newIdA, newIdB);
ItemUpdate update = new ItemUpdateBuilder(newIdA).addStatement(sNewAtoNewB).build();
assertEquals(QuickStatementsExporter.impossibleSchedulingErrorMessage,
export(update));
assertEquals(QuickStatementsExporter.impossibleSchedulingErrorMessage, export(update));
}
@Test
public void testNameDesc() throws IOException {
public void testNameDesc()
throws IOException {
ItemUpdate update = new ItemUpdateBuilder(newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("my new item", "en"))
.addDescription(Datamodel.makeMonolingualTextValue("isn't it awesome?", "en"))
.addAlias(Datamodel.makeMonolingualTextValue("fabitem", "en"))
.build();
.addAlias(Datamodel.makeMonolingualTextValue("fabitem", "en")).build();
assertEquals("CREATE\n"+
"LAST\tLen\t\"my new item\"\n"+
"LAST\tDen\t\"isn't it awesome?\"\n"+
"LAST\tAen\t\"fabitem\"\n",
export(update));
assertEquals("CREATE\n" + "LAST\tLen\t\"my new item\"\n" + "LAST\tDen\t\"isn't it awesome?\"\n"
+ "LAST\tAen\t\"fabitem\"\n", export(update));
}
@Test
public void testDeleteStatement() throws IOException {
ItemUpdate update = new ItemUpdateBuilder(qid1)
.deleteStatement(TestingData.generateStatement(qid1, qid2))
public void testDeleteStatement()
throws IOException {
ItemUpdate update = new ItemUpdateBuilder(qid1).deleteStatement(TestingData.generateStatement(qid1, qid2))
.build();
assertEquals("- Q1377\tP38\tQ865528\n", export(update));
}
@Test
public void testQualifier() throws IOException {
public void testQualifier()
throws IOException {
Statement baseStatement = TestingData.generateStatement(qid1, qid2);
Statement otherStatement = TestingData.generateStatement(qid2, qid1);
Snak qualifierSnak = otherStatement.getClaim().getMainSnak();
@ -100,15 +121,14 @@ public class QuickStatementsExporterTest extends RefineTest {
Claim claim = Datamodel.makeClaim(qid1, baseStatement.getClaim().getMainSnak(),
Collections.singletonList(group));
Statement statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
ItemUpdate update = new ItemUpdateBuilder(qid1)
.addStatement(statement)
.build();
ItemUpdate update = new ItemUpdateBuilder(qid1).addStatement(statement).build();
assertEquals("Q1377\tP38\tQ865528\tP38\tQ1377\n", export(update));
}
@Test
public void testNoSchema() throws IOException {
public void testNoSchema()
throws IOException {
Project project = this.createCSVProject("a,b\nc,d");
Engine engine = new Engine(project);
StringWriter writer = new StringWriter();

41
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java
View File

@ -1,5 +1,31 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.StringReader;
@ -12,9 +38,6 @@ import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
@ -42,12 +65,15 @@ public abstract class OperationTest extends RefineTest {
OperationRegistry.registerOperation(module, name, klass);
}
public abstract AbstractOperation reconstruct() throws Exception;
public abstract AbstractOperation reconstruct()
throws Exception;
public abstract JSONObject getJson() throws Exception;
public abstract JSONObject getJson()
throws Exception;
@Test
public void testReconstruct() throws Exception {
public void testReconstruct()
throws Exception {
JSONObject json = getJson();
AbstractOperation op = reconstruct();
StringWriter writer = new StringWriter();
@ -61,7 +87,8 @@ public abstract class OperationTest extends RefineTest {
return new LineNumberReader(reader);
}
protected String saveChange(Change change) throws IOException {
protected String saveChange(Change change)
throws IOException {
StringWriter writer = new StringWriter();
change.save(writer, new Properties());
return writer.toString();

35
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.junit.Assert.assertEquals;
@ -21,20 +44,22 @@ public class PerformWikibaseEditsOperationTest extends OperationTest {
}
@Override
public AbstractOperation reconstruct() throws Exception {
public AbstractOperation reconstruct()
throws Exception {
JSONObject json = getJson();
return PerformWikibaseEditsOperation.reconstruct(project, json);
}
@Override
public JSONObject getJson() throws Exception {
public JSONObject getJson()
throws Exception {
return TestingData.jsonFromFile("data/operations/perform-edits.json");
}
@Test
public void testLoadChange() throws Exception {
String changeString = "newItems={\"qidMap\":{\"1234\":\"Q789\"}}\n" +
"/ec/\n";
public void testLoadChange()
throws Exception {
String changeString = "newItems={\"qidMap\":{\"1234\":\"Q789\"}}\n" + "/ec/\n";
LineNumberReader reader = makeReader(changeString);
Change change = PerformWikibaseEditsOperation.PerformWikibaseEditsChange.load(reader, pool);

41
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import static org.junit.Assert.assertEquals;
@ -14,7 +37,6 @@ import org.testng.annotations.Test;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
public class SaveWikibaseSchemaOperationTest extends OperationTest {
@BeforeMethod
@ -23,22 +45,22 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest {
}
@Override
public AbstractOperation reconstruct() throws Exception {
public AbstractOperation reconstruct()
throws Exception {
return SaveWikibaseSchemaOperation.reconstruct(project, getJson());
}
@Override
public JSONObject getJson() throws Exception {
public JSONObject getJson()
throws Exception {
return TestingData.jsonFromFile("data/operations/save-schema.json");
}
@Test
public void testLoadChange() throws Exception {
public void testLoadChange()
throws Exception {
JSONObject schemaJson = TestingData.jsonFromFile("data/schema/inception.json");
String changeString =
"newSchema="+schemaJson.toString()+"\n" +
"oldSchema=\n" +
"/ec/";
String changeString = "newSchema=" + schemaJson.toString() + "\n" + "oldSchema=\n" + "/ec/";
WikibaseSchema schema = WikibaseSchema.reconstruct(schemaJson);
LineNumberReader reader = makeReader(changeString);
@ -46,7 +68,8 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest {
change.apply(project);
assertEquals(schema, project.overlayModels.get(SaveWikibaseSchemaOperation.WikibaseSchemaChange.overlayModelKey));
assertEquals(schema,
project.overlayModels.get(SaveWikibaseSchemaOperation.WikibaseSchemaChange.overlayModelKey));
change.revert(project);

28
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Arrays;
@ -8,7 +31,6 @@ import java.util.stream.Collectors;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class MockConstraintFetcher implements ConstraintFetcher {
public static PropertyIdValue pidWithInverse = Datamodel.makeWikidataPropertyIdValue("P350");
@ -26,8 +48,8 @@ public class MockConstraintFetcher implements ConstraintFetcher {
}
/**
* This constraint is purposely left inconsistent (the inverse
* constraint holds only on one side).
* This constraint is purposely left inconsistent (the inverse constraint holds
* only on one side).
*/
@Override
public PropertyIdValue getInversePid(PropertyIdValue pid) {

23
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/QAWarningStoreTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import static org.junit.Assert.assertEquals;

43
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/QAWarningTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import static org.junit.Assert.assertEquals;
@ -7,13 +30,10 @@ import org.testng.annotations.Test;
public class QAWarningTest {
public static QAWarning exampleWarning = new QAWarning("add-statements-with-invalid-format",
"P2427",
QAWarning.Severity.IMPORTANT,
1);
public static String exampleJson =
"{\"severity\":\"IMPORTANT\","+
"\"count\":1,\"bucketId\":\"P2427\",\"type\":\"add-statements-with-invalid-format\"}";
public static QAWarning exampleWarning = new QAWarning("add-statements-with-invalid-format", "P2427",
QAWarning.Severity.IMPORTANT, 1);
public static String exampleJson = "{\"severity\":\"IMPORTANT\","
+ "\"count\":1,\"bucketId\":\"P2427\",\"type\":\"add-statements-with-invalid-format\"}";
@Test
public void testSerialize() {
@ -22,9 +42,7 @@ public class QAWarningTest {
@Test
public void testAggregate() {
QAWarning firstWarning = new QAWarning("add-statements-with-invalid-format",
"P2427",
QAWarning.Severity.INFO,
QAWarning firstWarning = new QAWarning("add-statements-with-invalid-format", "P2427", QAWarning.Severity.INFO,
1);
firstWarning.setProperty("foo", "bar");
assertEquals(exampleWarning.getAggregationId(), firstWarning.getAggregationId());
@ -38,10 +56,7 @@ public class QAWarningTest {
@Test
public void testCompare() {
QAWarning otherWarning = new QAWarning("no-reference",
"no-reference",
QAWarning.Severity.WARNING,
1);
QAWarning otherWarning = new QAWarning("no-reference", "no-reference", QAWarning.Severity.WARNING, 1);
assertEquals(1, otherWarning.compareTo(exampleWarning));
assertEquals(-1, exampleWarning.compareTo(otherWarning));
assertEquals(0, exampleWarning.compareTo(exampleWarning));

23
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/WikidataConstraintFetcherTests.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import org.testng.Assert;

31
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.testing.TestingData;
@ -17,12 +40,8 @@ public class DistinctValuesScrutinizerTest extends StatementScrutinizerTest {
public void testTrigger() {
ItemIdValue idA = TestingData.existingId;
ItemIdValue idB = TestingData.matchedId;
ItemUpdate updateA = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, idB))
.build();
ItemUpdate updateB = new ItemUpdateBuilder(idB)
.addStatement(TestingData.generateStatement(idB, idB))
.build();
ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).build();
ItemUpdate updateB = new ItemUpdateBuilder(idB).addStatement(TestingData.generateStatement(idB, idB)).build();
scrutinize(updateA, updateB);
assertWarningsRaised(DistinctValuesScrutinizer.type);
}

23
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.testng.annotations.Test;

29
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.MockConstraintFetcher;
@ -23,16 +46,14 @@ public class InverseConstaintScrutinizerTest extends StatementScrutinizerTest {
@Test
public void testTrigger() {
ItemUpdate update = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, pidWithInverse, idB))
.build();
.addStatement(TestingData.generateStatement(idA, pidWithInverse, idB)).build();
scrutinize(update);
assertWarningsRaised(InverseConstraintScrutinizer.type);
}
@Test
public void testNoSymmetricClosure() {
ItemUpdate update = new ItemUpdateBuilder(idA)
.addStatement(TestingData.generateStatement(idA, inversePid, idB))
ItemUpdate update = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, inversePid, idB))
.build();
scrutinize(update);
assertNoWarningRaised();

41
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Collections;
@ -11,7 +34,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
public class NewItemScrutinizerTest extends ScrutinizerTest {
private Claim claim = Datamodel.makeClaim(TestingData.newIdA,
@ -28,11 +50,8 @@ public class NewItemScrutinizerTest extends ScrutinizerTest {
public void testTrigger() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA).build();
scrutinize(update);
assertWarningsRaised(
NewItemScrutinizer.noDescType,
NewItemScrutinizer.noLabelType,
NewItemScrutinizer.noTypeType,
NewItemScrutinizer.newItemType);
assertWarningsRaised(NewItemScrutinizer.noDescType, NewItemScrutinizer.noLabelType,
NewItemScrutinizer.noTypeType, NewItemScrutinizer.newItemType);
}
@Test
@ -47,8 +66,7 @@ public class NewItemScrutinizerTest extends ScrutinizerTest {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"))
.addStatement(p31Statement)
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType);
@ -58,11 +76,8 @@ public class NewItemScrutinizerTest extends ScrutinizerTest {
public void testDeletedStatements() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"))
.addStatement(p31Statement)
.deleteStatement(TestingData.generateStatement(TestingData.newIdA,
TestingData.matchedId))
.build();
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.deleteStatement(TestingData.generateStatement(TestingData.newIdA, TestingData.matchedId)).build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType, NewItemScrutinizer.deletedStatementsType);
}

23
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.testing.TestingData;

28
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizerTest.java
View File

@ -1,3 +1,26 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Arrays;
@ -16,6 +39,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
public class QualifierCompatibilityScrutinizerTest extends StatementScrutinizerTest {
private Snak disallowedQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.qualifierPid);
private Snak mandatoryQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.mandatoryQualifierPid);
private Snak allowedQualifier = Datamodel.makeNoValueSnak(MockConstraintFetcher.allowedQualifierPid);
@ -49,10 +73,10 @@ public class QualifierCompatibilityScrutinizerTest extends StatementScrutinizerT
Datamodel.makeNoValueSnak(MockConstraintFetcher.mainSnakPid), makeQualifiers(qualifiers));
return Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, "");
}
private List<SnakGroup> makeQualifiers(Snak[] qualifiers) {
List<Snak> snaks = Arrays.asList(qualifiers);
return snaks.stream()
.map((Snak q) -> Datamodel.makeSnakGroup(Collections.<Snak>singletonList(q)))
return snaks.stream().map((Snak q) -> Datamodel.makeSnakGroup(Collections.<Snak> singletonList(q)))
.collect(Collectors.toList());
}

Some files were not shown because too many files have changed in this diff Show More