s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make maxlag configurable (#2469)

Browse Source
This commit is contained in:
james-cui 2020-03-26 00:46:49 -07:00 committed by GitHub
parent 70b4c6a6d0
commit 7311581df0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -43,6 +43,9 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
/** /**
* Schedules and performs a list of updates to items via the API. * Schedules and performs a list of updates to items via the API.
@ -67,6 +70,17 @@ public class EditBatchProcessor {
private int globalCursor; private int globalCursor;
private Map<String, EntityDocument> currentDocs; private Map<String, EntityDocument> currentDocs;
private int batchSize; private int batchSize;
protected static final String MAX_LAG_KEY = "wikibase:upload:maxLag";
protected static final int MAX_LAG_DEFAULT = 5; // 5 second default maxLag
protected PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore();
private int getMaxLag() {
try {
return Integer.parseInt((String) prefStore.get(MAX_LAG_KEY));
} catch (NumberFormatException e) {
return MAX_LAG_DEFAULT;
}
}
/** /**
* Initiates the process of pushing a batch of updates to Wikibase. This * Initiates the process of pushing a batch of updates to Wikibase. This
@ -99,6 +113,9 @@ public class EditBatchProcessor {
// edit at 60 edits/min by default. If Wikidata is overloaded // edit at 60 edits/min by default. If Wikidata is overloaded
// it will slow us down via the maxlag mechanism. // it will slow us down via the maxlag mechanism.
editor.setAverageTimePerEdit(1000); editor.setAverageTimePerEdit(1000);
// set maxlag based on preference store
int maxLag = getMaxLag();
editor.setMaxLag(maxLag);
this.library = library; this.library = library;
this.summary = summary; this.summary = summary;

14
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java
View File

@ -52,6 +52,8 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
public class EditBatchProcessorTest extends WikidataRefineTest { public class EditBatchProcessorTest extends WikidataRefineTest {
@ -157,6 +159,18 @@ public class EditBatchProcessorTest extends WikidataRefineTest {
} }
} }
@Test
public void testSetMaxLag() {
// use default value
EditBatchProcessor processor1 = new EditBatchProcessor(fetcher, editor, Collections.emptyList(), library, summary, tags, 50);
verify(editor, times(1)).setMaxLag(EditBatchProcessor.MAX_LAG_DEFAULT);
// use value in preference store
ProjectManager.singleton.getPreferenceStore().put(EditBatchProcessor.MAX_LAG_KEY, "10");
EditBatchProcessor processor2 = new EditBatchProcessor(fetcher, editor, Collections.emptyList(), library, summary, tags, 50);
verify(editor, times(1)).setMaxLag(10);
}
private Map<String, EntityDocument> toMap(List<ItemDocument> docs) { private Map<String, EntityDocument> toMap(List<ItemDocument> docs) {
return docs.stream().collect(Collectors.toMap(doc -> doc.getEntityId().getId(), doc -> doc)); return docs.stream().collect(Collectors.toMap(doc -> doc.getEntityId().getId(), doc -> doc));
} }