diff --git a/main/src/com/google/refine/importers/WikitextImporter.java b/main/src/com/google/refine/importers/WikitextImporter.java index 06d6262ed..c36774c01 100644 --- a/main/src/com/google/refine/importers/WikitextImporter.java +++ b/main/src/com/google/refine/importers/WikitextImporter.java @@ -47,6 +47,7 @@ import xtc.parser.ParseException; import com.google.refine.ProjectMetadata; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Cell; +import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; @@ -388,7 +389,7 @@ public class WikitextImporter extends TabularImportingParserBase { return row; } - private void reconcileToQids(String wikiBaseUrl) { + private void reconcileToQids(String wikiBaseUrl, StandardReconConfig cfg) { if("null".equals(wikiBaseUrl)) { return; // TODO: more thorough URL validation instead } @@ -406,18 +407,6 @@ public class WikitextImporter extends TabularImportingParserBase { columnReconciled.add(false); } - // Wikidata reconciliation endpoint, hardcoded because the user might not have it in its services - StandardReconConfig cfg = new StandardReconConfig( - "https://tools.wmflabs.org/openrefine-wikidata/en/api", - "http://www.wikidata.org/entity/", - "http://www.wikidata.org/prop/direct/", - "", - "entity", - true, - new ArrayList(), - 1 - ); - int batchSize = 50; int i = 0; int totalSize = this.visitor.wikilinkedCells.size(); @@ -490,8 +479,13 @@ public class WikitextImporter extends TabularImportingParserBase { // Reconcile if needed String wikiUrl = JSONUtilities.getString(options, "wikiUrl", null); + // Wikidata reconciliation endpoint, hardcoded because the user might not have it in its services + String reconUrl = JSONUtilities.getString(options, "reconService", + "https://tools.wmflabs.org/openrefine-wikidata/en/api"); + StandardReconConfig cfg = getReconConfig(reconUrl); + if (wikiUrl != null) { - dataReader.reconcileToQids(wikiUrl); + dataReader.reconcileToQids(wikiUrl, cfg); } JSONUtilities.safePut(options, "headerLines", 1); @@ -508,7 +502,9 @@ public class WikitextImporter extends TabularImportingParserBase { if (dataReader.columnReconciled != null) { for(int i = 0; i != dataReader.columnReconciled.size(); i++) { if (dataReader.columnReconciled.get(i)) { - project.columnModel.columns.get(i).setReconStats(ReconStats.create(project, i)); + Column col = project.columnModel.columns.get(i); + col.setReconStats(ReconStats.create(project, i)); + col.setReconConfig(cfg); } } } @@ -520,5 +516,18 @@ public class WikitextImporter extends TabularImportingParserBase { } } + private StandardReconConfig getReconConfig(String url) { + StandardReconConfig cfg = new StandardReconConfig( + url, + "http://www.wikidata.org/entity/", + "http://www.wikidata.org/prop/direct/", + "", + "entity", + true, + new ArrayList(), + 1 + ); + return cfg; + } } diff --git a/main/tests/server/src/com/google/refine/tests/importers/WikitextImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/WikitextImporterTests.java index 742247d75..82b5ff7b0 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/WikitextImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/WikitextImporterTests.java @@ -204,6 +204,7 @@ public class WikitextImporterTests extends ImporterTest { whenGetBooleanOption("blankSpanningCells", options, blankSpanningCells); whenGetBooleanOption("storeBlankCellsAsNulls", options, true); whenGetIntegerOption("headerLines", options, 1); + whenGetStringOption("reconService", options, "https://tools.wmflabs.org/openrefine-wikidata/en/api"); } private void verifyOptions() { diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index ab4433840..ca9d45c0f 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -346,7 +346,9 @@ function init() { "scripts/index/parser-interfaces/xml-parser-ui.js", "scripts/index/parser-interfaces/json-parser-ui.js", "scripts/index/parser-interfaces/rdf-triples-parser-ui.js", - "scripts/index/parser-interfaces/wikitext-parser-ui.js" + "scripts/index/parser-interfaces/wikitext-parser-ui.js", + + "scripts/reconciliation/recon-manager.js" // so that reconciliation functions are available to importers ] ); diff --git a/main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.js b/main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.js index bdb5d5fdc..c64ff9116 100644 --- a/main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/extend-data-preview-dialog.js @@ -73,7 +73,7 @@ function ExtendReconciledDataPreviewDialog(column, columnIndex, rowIndices, onDo this._service = service; var serviceMetadata = ReconciliationManager.getServiceFromUrl(service); this._serviceMetadata = serviceMetadata; - if ("extend" in serviceMetadata) { + if (serviceMetadata != null && "extend" in serviceMetadata) { var extend = serviceMetadata.extend; if ("propose_properties" in extend) { var endpoint = extend.propose_properties; diff --git a/main/webapp/modules/core/scripts/index/parser-interfaces/wikitext-parser-ui.js b/main/webapp/modules/core/scripts/index/parser-interfaces/wikitext-parser-ui.js index 0bf9b7dc9..902845717 100644 --- a/main/webapp/modules/core/scripts/index/parser-interfaces/wikitext-parser-ui.js +++ b/main/webapp/modules/core/scripts/index/parser-interfaces/wikitext-parser-ui.js @@ -93,6 +93,8 @@ Refine.WikitextParserUI.prototype.getOptions = function() { options.storeBlankCellsAsNulls = this._optionContainerElmts.storeBlankCellsAsNullsCheckbox[0].checked; options.includeFileSources = this._optionContainerElmts.includeFileSourcesCheckbox[0].checked; + options.reconService = ReconciliationManager.ensureDefaultServicePresent(); + return options; }; diff --git a/main/webapp/modules/core/scripts/reconciliation/recon-manager.js b/main/webapp/modules/core/scripts/reconciliation/recon-manager.js index b3ed02c7b..36aab93bc 100644 --- a/main/webapp/modules/core/scripts/reconciliation/recon-manager.js +++ b/main/webapp/modules/core/scripts/reconciliation/recon-manager.js @@ -127,8 +127,28 @@ ReconciliationManager.save = function(f) { }); }; +ReconciliationManager.getOrRegisterServiceFromUrl = function(url, f) { + var service = ReconciliationManager.getServiceFromUrl(url); + if (service == null) { + ReconciliationManager.registerStandardService(url, function(idx) { + ReconciliationManager.save(function() { + f(ReconciliationManager.standardServices[i]); + }); + }); + } else { + f(service); + } +}; + +ReconciliationManager.ensureDefaultServicePresent = function() { + console.log('ensureDefaultServicePresent'); + var lang = $.i18n._('core-recon')["wd-recon-lang"]; + var url = "https://tools.wmflabs.org/openrefine-wikidata/"+lang+"/api"; + ReconciliationManager.getOrRegisterServiceFromUrl(url, function(service) { ; }); + return url; +}; + (function() { - var lang = $.i18n._('core-recon')["wd-recon-lang"]; $.ajax({ async: false, @@ -140,9 +160,7 @@ ReconciliationManager.save = function(f) { ReconciliationManager.standardServices = JSON.parse(data.value); ReconciliationManager._rebuildMap(); } else { - ReconciliationManager.registerStandardService( - "https://tools.wmflabs.org/openrefine-wikidata/"+lang+"/api" - ); + ReconciliationManager.ensureDefaultServicePresent(); } }, dataType: "json"