From 7311581df0327e6662ed818997f11f12e7c454c1 Mon Sep 17 00:00:00 2001 From: james-cui <61567716+james-cui@users.noreply.github.com> Date: Thu, 26 Mar 2020 00:46:49 -0700 Subject: [PATCH] make maxlag configurable (#2469) --- .../wikidata/editing/EditBatchProcessor.java | 17 +++++++++++++++++ .../editing/EditBatchProcessorTest.java | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java b/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java index c025ff5cf..a4f3699a1 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java @@ -43,6 +43,9 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; +import com.google.refine.ProjectManager; +import com.google.refine.preference.PreferenceStore; + /** * Schedules and performs a list of updates to items via the API. @@ -67,6 +70,17 @@ public class EditBatchProcessor { private int globalCursor; private Map currentDocs; private int batchSize; + protected static final String MAX_LAG_KEY = "wikibase:upload:maxLag"; + protected static final int MAX_LAG_DEFAULT = 5; // 5 second default maxLag + protected PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore(); + + private int getMaxLag() { + try { + return Integer.parseInt((String) prefStore.get(MAX_LAG_KEY)); + } catch (NumberFormatException e) { + return MAX_LAG_DEFAULT; + } + } /** * Initiates the process of pushing a batch of updates to Wikibase. This @@ -99,6 +113,9 @@ public class EditBatchProcessor { // edit at 60 edits/min by default. If Wikidata is overloaded // it will slow us down via the maxlag mechanism. editor.setAverageTimePerEdit(1000); + // set maxlag based on preference store + int maxLag = getMaxLag(); + editor.setMaxLag(maxLag); this.library = library; this.summary = summary; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java index 6cd3bfe62..519e1ecd0 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java @@ -52,6 +52,8 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; +import com.google.refine.ProjectManager; +import com.google.refine.preference.PreferenceStore; public class EditBatchProcessorTest extends WikidataRefineTest { @@ -157,6 +159,18 @@ public class EditBatchProcessorTest extends WikidataRefineTest { } } + @Test + public void testSetMaxLag() { + // use default value + EditBatchProcessor processor1 = new EditBatchProcessor(fetcher, editor, Collections.emptyList(), library, summary, tags, 50); + verify(editor, times(1)).setMaxLag(EditBatchProcessor.MAX_LAG_DEFAULT); + + // use value in preference store + ProjectManager.singleton.getPreferenceStore().put(EditBatchProcessor.MAX_LAG_KEY, "10"); + EditBatchProcessor processor2 = new EditBatchProcessor(fetcher, editor, Collections.emptyList(), library, summary, tags, 50); + verify(editor, times(1)).setMaxLag(10); + } + private Map toMap(List docs) { return docs.stream().collect(Collectors.toMap(doc -> doc.getEntityId().getId(), doc -> doc)); }