s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2208 from OpenRefine/issue-2205-scheduling-error

Browse Source 
Change behaviour of Wikidata editing in case of network failure
This commit is contained in:
Antonin Delpeuch 2019-11-10 17:38:47 +00:00 committed by GitHub
commit 90bf6ecd70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -169,9 +169,10 @@ public class EditBatchProcessor {
} }
} catch (MediaWikiApiErrorException e) { } catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way // TODO find a way to report these errors to the user in a nice way
e.printStackTrace(); logger.warn("MediaWiki error while editing [" + e.getErrorCode()
+ "]: " + e.getErrorMessage());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.warn("IO error while editing: " + e.getMessage());
} }
batchCursor++; batchCursor++;
@ -209,22 +210,29 @@ public class EditBatchProcessor {
// Get the current documents for this batch of updates // Get the current documents for this batch of updates
logger.info("Requesting documents"); logger.info("Requesting documents");
currentDocs = null; currentDocs = null;
int retries = 3; int retries = 5;
int backoff = 2;
int sleepTime = 5000;
// TODO: remove currentDocs.isEmpty() once https://github.com/Wikidata/Wikidata-Toolkit/issues/402 is solved // TODO: remove currentDocs.isEmpty() once https://github.com/Wikidata/Wikidata-Toolkit/issues/402 is solved
while ((currentDocs == null || currentDocs.isEmpty()) && retries > 0) { while ((currentDocs == null || currentDocs.isEmpty()) && retries > 0) {
try { try {
currentDocs = fetcher.getEntityDocuments(qidsToFetch); currentDocs = fetcher.getEntityDocuments(qidsToFetch);
} catch (MediaWikiApiErrorException e) { } catch (MediaWikiApiErrorException e) {
e.printStackTrace(); logger.warn("MediaWiki error while fetching documents to edit [" + e.getErrorCode()
Thread.sleep(5000); + "]: " + e.getErrorMessage());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.warn("IO error while fetching documents to edit: " + e.getMessage());
Thread.sleep(5000);
} }
retries--; retries--;
sleepTime *= backoff;
if ((currentDocs == null || currentDocs.isEmpty()) && retries > 0) {
logger.warn("Retrying in " + sleepTime + " ms");
Thread.sleep(sleepTime);
}
} }
if (currentDocs == null) { if (currentDocs == null) {
throw new InterruptedException("Fetching current documents failed."); logger.warn("Giving up on fetching documents to edit. Skipping "+remainingEdits()+" remaining edits.");
globalCursor = scheduled.size();
} }
batchCursor = 0; batchCursor = 0;
} }