Update Wikidata-Toolkit to solve various issues (#2215)
Updates Wikidata-Toolkit to 0.11.0 This enables the following improvements: - https://github.com/Wikidata/Wikidata-Toolkit/issues/431 - https://github.com/Wikidata/Wikidata-Toolkit/issues/456 This closes #2196 although the feature will not be enabled until 3.4 for technical reasons (the openrefine-3.3 tag on Wikidata can only be applied by an AbuseFilter, not manually through the API). * Update WDTK and add timeouts to Wikidata editing, closes #2211. * Explicitly add wdtk-util as a dependency * Catch MediaWikiApiError following change of interface
This commit is contained in:
parent
b561824d04
commit
b19e8a9a74
@ -128,12 +128,17 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.wikidata.wdtk</groupId>
|
<groupId>org.wikidata.wdtk</groupId>
|
||||||
<artifactId>wdtk-wikibaseapi</artifactId>
|
<artifactId>wdtk-wikibaseapi</artifactId>
|
||||||
<version>0.10.0</version>
|
<version>0.11.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.wikidata.wdtk</groupId>
|
<groupId>org.wikidata.wdtk</groupId>
|
||||||
<artifactId>wdtk-datamodel</artifactId>
|
<artifactId>wdtk-datamodel</artifactId>
|
||||||
<version>0.10.0</version>
|
<version>0.11.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.wikidata.wdtk</groupId>
|
||||||
|
<artifactId>wdtk-util</artifactId>
|
||||||
|
<version>0.11.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
|
@ -28,7 +28,9 @@ import java.io.IOException;
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
||||||
|
import org.wikidata.wdtk.wikibaseapi.BasicApiConnection;
|
||||||
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
|
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
|
||||||
|
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
@ -54,9 +56,11 @@ public class ConnectionManager {
|
|||||||
final static Logger logger = LoggerFactory.getLogger("connection_mananger");
|
final static Logger logger = LoggerFactory.getLogger("connection_mananger");
|
||||||
|
|
||||||
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
|
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
|
||||||
|
public static final int CONNECT_TIMEOUT = 5000;
|
||||||
|
public static final int READ_TIMEOUT = 10000;
|
||||||
|
|
||||||
private PreferenceStore prefStore;
|
private PreferenceStore prefStore;
|
||||||
private ApiConnection connection;
|
private BasicApiConnection connection;
|
||||||
|
|
||||||
private static final ConnectionManager instance = new ConnectionManager();
|
private static final ConnectionManager instance = new ConnectionManager();
|
||||||
|
|
||||||
@ -64,12 +68,26 @@ public class ConnectionManager {
|
|||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a connection manager, which attempts to restore any
|
||||||
|
* previous connection (from the preferences).
|
||||||
|
*/
|
||||||
private ConnectionManager() {
|
private ConnectionManager() {
|
||||||
prefStore = ProjectManager.singleton.getPreferenceStore();
|
prefStore = ProjectManager.singleton.getPreferenceStore();
|
||||||
connection = null;
|
connection = null;
|
||||||
restoreSavedConnection();
|
restoreSavedConnection();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Logs in to the Wikibase instance, using login/password
|
||||||
|
*
|
||||||
|
* @param username
|
||||||
|
* the username to log in with
|
||||||
|
* @param password
|
||||||
|
* the password to log in with
|
||||||
|
* @param rememberCredentials
|
||||||
|
* whether to store these credentials in the preferences (unencrypted!)
|
||||||
|
*/
|
||||||
public void login(String username, String password, boolean rememberCredentials) {
|
public void login(String username, String password, boolean rememberCredentials) {
|
||||||
if (rememberCredentials) {
|
if (rememberCredentials) {
|
||||||
ArrayNode array = ParsingUtilities.mapper.createArrayNode();
|
ArrayNode array = ParsingUtilities.mapper.createArrayNode();
|
||||||
@ -80,7 +98,7 @@ public class ConnectionManager {
|
|||||||
prefStore.put(PREFERENCE_STORE_KEY, array);
|
prefStore.put(PREFERENCE_STORE_KEY, array);
|
||||||
}
|
}
|
||||||
|
|
||||||
connection = ApiConnection.getWikidataApiConnection();
|
connection = createNewConnection();
|
||||||
try {
|
try {
|
||||||
connection.login(username, password);
|
connection.login(username, password);
|
||||||
} catch (LoginFailedException e) {
|
} catch (LoginFailedException e) {
|
||||||
@ -88,10 +106,13 @@ public class ConnectionManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restore any previously saved connection, from the preferences.
|
||||||
|
*/
|
||||||
public void restoreSavedConnection() {
|
public void restoreSavedConnection() {
|
||||||
ObjectNode savedCredentials = getStoredCredentials();
|
ObjectNode savedCredentials = getStoredCredentials();
|
||||||
if (savedCredentials != null) {
|
if (savedCredentials != null) {
|
||||||
connection = ApiConnection.getWikidataApiConnection();
|
connection = createNewConnection();
|
||||||
try {
|
try {
|
||||||
connection.login(savedCredentials.get("username").asText(), savedCredentials.get("password").asText());
|
connection.login(savedCredentials.get("username").asText(), savedCredentials.get("password").asText());
|
||||||
} catch (LoginFailedException e) {
|
} catch (LoginFailedException e) {
|
||||||
@ -114,7 +135,7 @@ public class ConnectionManager {
|
|||||||
try {
|
try {
|
||||||
connection.logout();
|
connection.logout();
|
||||||
connection = null;
|
connection = null;
|
||||||
} catch (IOException e) {
|
} catch (IOException | MediaWikiApiErrorException e) {
|
||||||
logger.error(e.getMessage());
|
logger.error(e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -135,4 +156,16 @@ public class ConnectionManager {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a fresh connection object with our
|
||||||
|
* prefered settings.
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
protected BasicApiConnection createNewConnection() {
|
||||||
|
BasicApiConnection conn = BasicApiConnection.getWikidataApiConnection();
|
||||||
|
conn.setConnectTimeout(CONNECT_TIMEOUT);
|
||||||
|
conn.setReadTimeout(READ_TIMEOUT);
|
||||||
|
return conn;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -59,6 +59,7 @@ public class EditBatchProcessor {
|
|||||||
private NewItemLibrary library;
|
private NewItemLibrary library;
|
||||||
private List<ItemUpdate> scheduled;
|
private List<ItemUpdate> scheduled;
|
||||||
private String summary;
|
private String summary;
|
||||||
|
private List<String> tags;
|
||||||
|
|
||||||
private List<ItemUpdate> remainingUpdates;
|
private List<ItemUpdate> remainingUpdates;
|
||||||
private List<ItemUpdate> currentBatch;
|
private List<ItemUpdate> currentBatch;
|
||||||
@ -82,12 +83,14 @@ public class EditBatchProcessor {
|
|||||||
* the library to use to keep track of new item creation
|
* the library to use to keep track of new item creation
|
||||||
* @param summary
|
* @param summary
|
||||||
* the summary to append to all edits
|
* the summary to append to all edits
|
||||||
|
* @param tags
|
||||||
|
* the list of tags to apply to all edits
|
||||||
* @param batchSize
|
* @param batchSize
|
||||||
* the number of items that should be retrieved in one go from the
|
* the number of items that should be retrieved in one go from the
|
||||||
* API
|
* API
|
||||||
*/
|
*/
|
||||||
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
|
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
|
||||||
NewItemLibrary library, String summary, int batchSize) {
|
NewItemLibrary library, String summary, List<String> tags, int batchSize) {
|
||||||
this.fetcher = fetcher;
|
this.fetcher = fetcher;
|
||||||
this.editor = editor;
|
this.editor = editor;
|
||||||
editor.setEditAsBot(true); // this will not do anything if the user does not
|
editor.setEditAsBot(true); // this will not do anything if the user does not
|
||||||
@ -99,6 +102,7 @@ public class EditBatchProcessor {
|
|||||||
|
|
||||||
this.library = library;
|
this.library = library;
|
||||||
this.summary = summary;
|
this.summary = summary;
|
||||||
|
this.tags = tags;
|
||||||
this.batchSize = batchSize;
|
this.batchSize = batchSize;
|
||||||
|
|
||||||
// Schedule the edit batch
|
// Schedule the edit batch
|
||||||
@ -133,6 +137,7 @@ public class EditBatchProcessor {
|
|||||||
update = rewriter.rewrite(update);
|
update = rewriter.rewrite(update);
|
||||||
} catch (NewItemNotCreatedYetException e) {
|
} catch (NewItemNotCreatedYetException e) {
|
||||||
logger.warn("Failed to rewrite update on entity "+update.getItemId()+". Missing entity: "+e.getMissingEntity()+". Skipping update.");
|
logger.warn("Failed to rewrite update on entity "+update.getItemId()+". Missing entity: "+e.getMissingEntity()+". Skipping update.");
|
||||||
|
batchCursor++;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -148,8 +153,8 @@ public class EditBatchProcessor {
|
|||||||
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
|
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
|
||||||
Collections.emptyMap());
|
Collections.emptyMap());
|
||||||
|
|
||||||
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
|
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary, tags);
|
||||||
library.setQid(newCell.getReconInternalId(), createdDoc.getItemId().getId());
|
library.setQid(newCell.getReconInternalId(), createdDoc.getEntityId().getId());
|
||||||
} else {
|
} else {
|
||||||
// Existing item
|
// Existing item
|
||||||
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
|
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
|
||||||
@ -165,7 +170,8 @@ public class EditBatchProcessor {
|
|||||||
update.getAliases().stream().collect(Collectors.toList()),
|
update.getAliases().stream().collect(Collectors.toList()),
|
||||||
new ArrayList<MonolingualTextValue>(),
|
new ArrayList<MonolingualTextValue>(),
|
||||||
update.getAddedStatements().stream().collect(Collectors.toList()),
|
update.getAddedStatements().stream().collect(Collectors.toList()),
|
||||||
update.getDeletedStatements().stream().collect(Collectors.toList()), summary);
|
update.getDeletedStatements().stream().collect(Collectors.toList()),
|
||||||
|
summary, tags);
|
||||||
}
|
}
|
||||||
} catch (MediaWikiApiErrorException e) {
|
} catch (MediaWikiApiErrorException e) {
|
||||||
// TODO find a way to report these errors to the user in a nice way
|
// TODO find a way to report these errors to the user in a nice way
|
||||||
|
@ -28,7 +28,6 @@ import java.util.Locale;
|
|||||||
|
|
||||||
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
|
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
|
||||||
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
|
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
|
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||||
@ -36,6 +35,7 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
|
|||||||
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
|
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
|
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
||||||
|
import org.wikidata.wdtk.datamodel.interfaces.UnsupportedValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
|
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -96,4 +96,10 @@ public class QSValuePrinter implements ValueVisitor<String> {
|
|||||||
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
|
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
|
||||||
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
|
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String visit(UnsupportedValue value) {
|
||||||
|
// we know this cannot happen, since UnsupportedValues cannot be generated in OpenRefine
|
||||||
|
return "<UNSUPPORTED>";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -26,11 +26,16 @@ package org.openrefine.wikidata.operations;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.LineNumberReader;
|
import java.io.LineNumberReader;
|
||||||
import java.io.Writer;
|
import java.io.Writer;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
import org.apache.commons.lang.Validate;
|
import org.apache.commons.lang.Validate;
|
||||||
|
import org.apache.log4j.spi.LoggerRepository;
|
||||||
import org.openrefine.wikidata.editing.ConnectionManager;
|
import org.openrefine.wikidata.editing.ConnectionManager;
|
||||||
import org.openrefine.wikidata.editing.EditBatchProcessor;
|
import org.openrefine.wikidata.editing.EditBatchProcessor;
|
||||||
import org.openrefine.wikidata.editing.NewItemLibrary;
|
import org.openrefine.wikidata.editing.NewItemLibrary;
|
||||||
@ -46,6 +51,7 @@ import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
|
|||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.refine.RefineServlet;
|
||||||
import com.google.refine.browsing.Engine;
|
import com.google.refine.browsing.Engine;
|
||||||
import com.google.refine.browsing.EngineConfig;
|
import com.google.refine.browsing.EngineConfig;
|
||||||
import com.google.refine.history.Change;
|
import com.google.refine.history.Change;
|
||||||
@ -143,6 +149,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
|
|||||||
protected Engine _engine;
|
protected Engine _engine;
|
||||||
protected WikibaseSchema _schema;
|
protected WikibaseSchema _schema;
|
||||||
protected String _summary;
|
protected String _summary;
|
||||||
|
protected List<String> _tags;
|
||||||
protected final long _historyEntryID;
|
protected final long _historyEntryID;
|
||||||
|
|
||||||
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
|
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
|
||||||
@ -151,6 +158,15 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
|
|||||||
this._engine = engine;
|
this._engine = engine;
|
||||||
this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
|
this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
|
||||||
this._summary = summary;
|
this._summary = summary;
|
||||||
|
// TODO this is one of the attributes that should be configured on a per-wiki basis
|
||||||
|
// TODO enable this tag once 3.3 final is released and create 3.4 tag without AbuseFilter
|
||||||
|
String tag = "openrefine";
|
||||||
|
Pattern pattern = Pattern.compile("^(\\d+\\.\\d+).*$");
|
||||||
|
Matcher matcher = pattern.matcher(RefineServlet.VERSION);
|
||||||
|
if (matcher.matches()) {
|
||||||
|
tag += "-"+matcher.group(1);
|
||||||
|
}
|
||||||
|
this._tags = Collections.emptyList(); // TODO Arrays.asList(tag);
|
||||||
this._historyEntryID = HistoryEntry.allocateID();
|
this._historyEntryID = HistoryEntry.allocateID();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -183,7 +199,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
|
|||||||
// Prepare the edits
|
// Prepare the edits
|
||||||
NewItemLibrary newItemLibrary = new NewItemLibrary();
|
NewItemLibrary newItemLibrary = new NewItemLibrary();
|
||||||
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, summary,
|
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, summary,
|
||||||
50);
|
_tags, 50);
|
||||||
|
|
||||||
// Perform edits
|
// Perform edits
|
||||||
logger.info("Performing edits");
|
logger.info("Performing edits");
|
||||||
|
@ -1,10 +1,8 @@
|
|||||||
package org.openrefine.wikidata.qa.scrutinizers;
|
package org.openrefine.wikidata.qa.scrutinizers;
|
||||||
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import org.openrefine.wikidata.qa.QAWarning;
|
import org.openrefine.wikidata.qa.QAWarning;
|
||||||
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
||||||
|
@ -28,7 +28,6 @@ import java.util.List;
|
|||||||
|
|
||||||
import org.jsoup.helper.Validate;
|
import org.jsoup.helper.Validate;
|
||||||
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
|
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
|
||||||
import org.openrefine.wikidata.utils.StatementGroupJson;
|
|
||||||
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
||||||
|
@ -32,12 +32,10 @@ import java.util.LinkedList;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.jsoup.helper.Validate;
|
import org.jsoup.helper.Validate;
|
||||||
import org.openrefine.wikidata.utils.StatementGroupJson;
|
import org.openrefine.wikidata.utils.StatementGroupJson;
|
||||||
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
|
||||||
import org.wikidata.wdtk.datamodel.implementation.StatementGroupImpl;
|
import org.wikidata.wdtk.datamodel.implementation.StatementGroupImpl;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
|
||||||
|
@ -29,7 +29,6 @@ import java.util.List;
|
|||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
|
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
|
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
|
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
|
||||||
@ -39,6 +38,7 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
|
|||||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
|
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
||||||
|
import org.wikidata.wdtk.datamodel.interfaces.UnsupportedValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.Value;
|
import org.wikidata.wdtk.datamodel.interfaces.Value;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
|
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
|
||||||
|
|
||||||
@ -158,4 +158,9 @@ public class PointerExtractor implements ValueVisitor<Set<ReconItemIdValue>> {
|
|||||||
public Set<ReconItemIdValue> visit(TimeValue value) {
|
public Set<ReconItemIdValue> visit(TimeValue value) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<ReconItemIdValue> visit(UnsupportedValue value) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,6 +29,7 @@ import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
|||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
|
||||||
|
import org.wikidata.wdtk.wikibaseapi.BasicApiConnection;
|
||||||
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
|
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
|
||||||
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
|
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
|
||||||
|
|
||||||
@ -44,7 +45,7 @@ public class EntityCache {
|
|||||||
private WikibaseDataFetcher _fetcher;
|
private WikibaseDataFetcher _fetcher;
|
||||||
|
|
||||||
private EntityCache() {
|
private EntityCache() {
|
||||||
ApiConnection connection = ApiConnection.getWikidataApiConnection();
|
ApiConnection connection = BasicApiConnection.getWikidataApiConnection();
|
||||||
_fetcher = new WikibaseDataFetcher(connection, Datamodel.SITE_WIKIDATA);
|
_fetcher = new WikibaseDataFetcher(connection, Datamodel.SITE_WIKIDATA);
|
||||||
|
|
||||||
_cache = CacheBuilder.newBuilder().maximumSize(4096).expireAfterWrite(1, TimeUnit.HOURS)
|
_cache = CacheBuilder.newBuilder().maximumSize(4096).expireAfterWrite(1, TimeUnit.HOURS)
|
||||||
|
@ -2,7 +2,6 @@ package org.openrefine.wikidata.utils;
|
|||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
package org.openrefine.wikidata.commands;
|
package org.openrefine.wikidata.commands;
|
||||||
|
|
||||||
import static org.testng.Assert.assertEquals;
|
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -23,9 +23,9 @@
|
|||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
package org.openrefine.wikidata.commands;
|
package org.openrefine.wikidata.commands;
|
||||||
|
|
||||||
import static org.testng.Assert.assertEquals;
|
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
|
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
|
||||||
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -23,9 +23,9 @@
|
|||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
package org.openrefine.wikidata.commands;
|
package org.openrefine.wikidata.commands;
|
||||||
|
|
||||||
import static org.testng.Assert.assertTrue;
|
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
|
import static org.openrefine.wikidata.testing.TestingData.jsonFromFile;
|
||||||
|
import static org.testng.Assert.assertTrue;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -23,8 +23,8 @@
|
|||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
package org.openrefine.wikidata.commands;
|
package org.openrefine.wikidata.commands;
|
||||||
|
|
||||||
import static org.testng.Assert.assertEquals;
|
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@ import static org.testng.Assert.assertEquals;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@ -58,6 +59,7 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
|
|||||||
private WikibaseDataEditor editor = null;
|
private WikibaseDataEditor editor = null;
|
||||||
private NewItemLibrary library = null;
|
private NewItemLibrary library = null;
|
||||||
private String summary = "my fantastic edits";
|
private String summary = "my fantastic edits";
|
||||||
|
private List<String> tags = null;
|
||||||
|
|
||||||
@BeforeMethod
|
@BeforeMethod
|
||||||
public void setUp() {
|
public void setUp() {
|
||||||
@ -65,6 +67,7 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
|
|||||||
editor = mock(WikibaseDataEditor.class);
|
editor = mock(WikibaseDataEditor.class);
|
||||||
editor.disableEditing(); // just in case we got mocking wrong…
|
editor.disableEditing(); // just in case we got mocking wrong…
|
||||||
library = new NewItemLibrary();
|
library = new NewItemLibrary();
|
||||||
|
tags = Arrays.asList("my-tag");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -87,9 +90,9 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
|
|||||||
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
|
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
|
||||||
ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234"))
|
ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234"))
|
||||||
.withLabel(label).withRevisionId(37828L).build();
|
.withLabel(label).withRevisionId(37828L).build();
|
||||||
when(editor.createItemDocument(expectedNewItem, summary)).thenReturn(createdNewItem);
|
when(editor.createItemDocument(expectedNewItem, summary, tags)).thenReturn(createdNewItem);
|
||||||
|
|
||||||
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, 50);
|
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, tags, 50);
|
||||||
assertEquals(2, processor.remainingEdits());
|
assertEquals(2, processor.remainingEdits());
|
||||||
assertEquals(0, processor.progress());
|
assertEquals(0, processor.progress());
|
||||||
processor.performEdit();
|
processor.performEdit();
|
||||||
@ -134,7 +137,7 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
|
|||||||
when(fetcher.getEntityDocuments(toQids(secondBatch))).thenReturn(toMap(secondBatch));
|
when(fetcher.getEntityDocuments(toQids(secondBatch))).thenReturn(toMap(secondBatch));
|
||||||
|
|
||||||
// Run edits
|
// Run edits
|
||||||
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, batchSize);
|
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, tags, batchSize);
|
||||||
assertEquals(0, processor.progress());
|
assertEquals(0, processor.progress());
|
||||||
for (int i = 124; i < 190; i++) {
|
for (int i = 124; i < 190; i++) {
|
||||||
assertEquals(processor.remainingEdits(), 190 - i);
|
assertEquals(processor.remainingEdits(), 190 - i);
|
||||||
@ -150,15 +153,15 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
|
|||||||
for (ItemDocument doc : fullBatch) {
|
for (ItemDocument doc : fullBatch) {
|
||||||
verify(editor, times(1)).updateTermsStatements(doc, Collections.emptyList(),
|
verify(editor, times(1)).updateTermsStatements(doc, Collections.emptyList(),
|
||||||
Collections.singletonList(description), Collections.emptyList(), Collections.emptyList(),
|
Collections.singletonList(description), Collections.emptyList(), Collections.emptyList(),
|
||||||
Collections.emptyList(), Collections.emptyList(), summary);
|
Collections.emptyList(), Collections.emptyList(), summary, tags);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, EntityDocument> toMap(List<ItemDocument> docs) {
|
private Map<String, EntityDocument> toMap(List<ItemDocument> docs) {
|
||||||
return docs.stream().collect(Collectors.toMap(doc -> doc.getItemId().getId(), doc -> doc));
|
return docs.stream().collect(Collectors.toMap(doc -> doc.getEntityId().getId(), doc -> doc));
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<String> toQids(List<ItemDocument> docs) {
|
private List<String> toQids(List<ItemDocument> docs) {
|
||||||
return docs.stream().map(doc -> doc.getItemId().getId()).collect(Collectors.toList());
|
return docs.stream().map(doc -> doc.getEntityId().getId()).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -34,7 +34,6 @@ import org.openrefine.wikidata.testing.WikidataRefineTest;
|
|||||||
import org.testng.annotations.BeforeMethod;
|
import org.testng.annotations.BeforeMethod;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
import com.google.refine.RefineTest;
|
|
||||||
import com.google.refine.model.Cell;
|
import com.google.refine.model.Cell;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
import com.google.refine.model.Recon;
|
import com.google.refine.model.Recon;
|
||||||
|
@ -45,7 +45,6 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
|
|||||||
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
import org.wikidata.wdtk.datamodel.interfaces.Statement;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
|
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
|
||||||
|
|
||||||
import com.google.refine.RefineTest;
|
|
||||||
import com.google.refine.browsing.Engine;
|
import com.google.refine.browsing.Engine;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
|
|
||||||
|
@ -7,7 +7,6 @@ import java.util.Properties;
|
|||||||
import org.openrefine.wikidata.testing.WikidataRefineTest;
|
import org.openrefine.wikidata.testing.WikidataRefineTest;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
import com.google.refine.RefineTest;
|
|
||||||
import com.google.refine.browsing.Engine;
|
import com.google.refine.browsing.Engine;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
import com.google.refine.util.TestUtils;
|
import com.google.refine.util.TestUtils;
|
||||||
|
@ -23,6 +23,7 @@
|
|||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
package org.openrefine.wikidata.updates.scheduler;
|
package org.openrefine.wikidata.updates.scheduler;
|
||||||
|
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.testng.Assert.assertEquals;
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
@ -41,6 +42,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak;
|
|||||||
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
|
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
|
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
|
||||||
|
import org.wikidata.wdtk.datamodel.interfaces.UnsupportedValue;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.Value;
|
import org.wikidata.wdtk.datamodel.interfaces.Value;
|
||||||
|
|
||||||
public class PointerExtractorTest {
|
public class PointerExtractorTest {
|
||||||
@ -73,6 +75,7 @@ public class PointerExtractorTest {
|
|||||||
assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("898")));
|
assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("898")));
|
||||||
assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("7.87"), "http://www.wikidata.org/entity/Q34"));
|
assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("7.87"), "http://www.wikidata.org/entity/Q34"));
|
||||||
assertEmpty(Datamodel.makeTimeValue(1898, (byte) 2, (byte) 3, TimeValue.CM_GREGORIAN_PRO));
|
assertEmpty(Datamodel.makeTimeValue(1898, (byte) 2, (byte) 3, TimeValue.CM_GREGORIAN_PRO));
|
||||||
|
assertEmpty(mock(UnsupportedValue.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
Loading…
Reference in New Issue
Block a user