Merge branch 'master' of https://github.com/OpenRefine/OpenRefine

README.md

@@ -1,6 +1,6 @@
 # OpenRefine
 
-[![Build Status](https://travis-ci.org/OpenRefine/OpenRefine.png?branch=master)](https://travis-ci.org/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Translation progress](https://hosted.weblate.org/widgets/openrefine/-/svg-badge.svg)](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=32795)](https://www.bountysource.com/trackers/32795-open-refine?utm_source=32795&utm_medium=shield&utm_campaign=TRACKER_BADGE)
+[![Join the chat at https://gitter.im/OpenRefine/OpenRefine](https://badges.gitter.im/OpenRefine/OpenRefine.svg)](https://gitter.im/OpenRefine/OpenRefine?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/OpenRefine/OpenRefine.png?branch=master)](https://travis-ci.org/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Translation progress](https://hosted.weblate.org/widgets/openrefine/-/svg-badge.svg)](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=32795)](https://www.bountysource.com/trackers/32795-open-refine?utm_source=32795&utm_medium=shield&utm_campaign=TRACKER_BADGE)
 
 OpenRefine is a power tool that allows you to load data, understand it,
 clean it up, reconcile it, and augment it with data coming from

main/src/com/google/refine/commands/recon/ReconJudgeOneCellCommand.java

@@ -193,7 +193,7 @@ public class ReconJudgeOneCellCommand extends Command {
                 newCell.recon.judgment = Recon.Judgment.New;
                 newCell.recon.match = null;
 
-                description = "Mark to create new topic for " + cellDescription;
+                description = "Mark to create new item for " + cellDescription;
             } else {
                 newCell.recon.judgment = Recon.Judgment.Matched;
                 newCell.recon.match = this.match;

main/src/com/google/refine/importers/WikitextImporter.java

@@ -156,7 +156,7 @@ public class WikitextImporter extends TabularImportingParserBase {
         private int spanningCellIdx;
         private List<String> internalLinksInCell;
         
-        private final Pattern urlPattern = Pattern.compile("\\b(https?|ftp)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]",
+        private final Pattern urlPattern = Pattern.compile("\\b(https?|ftp)://[-a-zA-Z0-9+&@#/%?=~_!:,.;]*[-a-zA-Z0-9+&@#/%=~_]",
                 Pattern.CASE_INSENSITIVE);
         
         public WikitextTableVisitor(boolean blankSpanningCells, boolean includeRawTemplates) {

main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java

@@ -142,14 +142,14 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
                 _similarValue + "\" in column " + _columnName;
         } else if (_judgment == Judgment.New) {
             if (_shareNewTopics) {
-                return "Mark to create one single new topic for all cells containing \"" +
+                return "Mark to create one single new item for all cells containing \"" +
                     _similarValue + "\" in column " + _columnName;
             } else {
-                return "Mark to create one new topic for each cell containing \"" +
+                return "Mark to create one new item for each cell containing \"" +
                     _similarValue + "\" in column " + _columnName;
             }
         } else if (_judgment == Judgment.Matched) {
-            return "Match topic " + 
+            return "Match item " + 
                 _match.name +  " (" +
                 _match.id + ") for cells containing \"" +
                 _similarValue + "\" in column " + _columnName;
@@ -166,14 +166,14 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
                 _similarValue + "\" in column " + _columnName;
         } else if (_judgment == Judgment.New) {
             if (_shareNewTopics) {
-                return "Mark to create one single new topic for " + cellChanges.size() + " cells containing \"" +
+                return "Mark to create one single new item for " + cellChanges.size() + " cells containing \"" +
                     _similarValue + "\" in column " + _columnName;
             } else {
-                return "Mark to create one new topic for each of " + cellChanges.size() + " cells containing \"" +
+                return "Mark to create one new item for each of " + cellChanges.size() + " cells containing \"" +
                     _similarValue + "\" in column " + _columnName;
             }
         } else if (_judgment == Judgment.Matched) {
-            return "Match topic " + 
+            return "Match item " + 
                 _match.name + " (" +
                 _match.id + ") for " +
                 cellChanges.size() + " cells containing \"" +

main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java

@@ -89,21 +89,21 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
     
     @Override
     protected String getBriefDescription(Project project) {
-        return "Mark to create new topics for cells in column " + _columnName +
+        return "Mark to create new items for cells in column " + _columnName +
             (_shareNewTopics ? 
-                ", one topic for each group of similar cells" : 
+                ", one item for each group of similar cells" : 
-                ", one topic for each cell");
+                ", one item for each cell");
     }
 
     @Override
     protected String createDescription(Column column,
             List<CellChange> cellChanges) {
         
-        return "Mark to create new topics for " + cellChanges.size() + 
+        return "Mark to create new items for " + cellChanges.size() + 
             " cells in column " + column.getName() +
             (_shareNewTopics ? 
-                ", one topic for each group of similar cells" : 
+                ", one item for each group of similar cells" : 
-                ", one topic for each cell");
+                ", one item for each cell");
     }
 
     @Override

main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java

@@ -128,7 +128,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
     
     @Override
     protected String getBriefDescription(Project project) {
-        return "Match specific topic " +
+        return "Match specific item " +
             match.name + " (" + 
             match.id + ") to cells in column " + _columnName;
     }
@@ -136,7 +136,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
     @Override
     protected String createDescription(Column column,
             List<CellChange> cellChanges) {
-        return "Match specific topic " + 
+        return "Match specific item " + 
             match.name + " (" + 
             match.id + ") to " + cellChanges.size() + 
             " cells in column " + column.getName();

main/tests/server/src/com/google/refine/tests/importers/WikitextImporterTests.java

@@ -224,6 +224,35 @@ public class WikitextImporterTests extends ImporterTest {
         Assert.assertEquals(project.rows.get(1).cells.get(4).value, "http://gnu.org");
         Assert.assertEquals(project.rows.get(1).cells.get(2).value, "http://microsoft.com/");
     }
+
+    @Test
+    public void readTableWithReferencesTemplates() {
+        // inspired from https://www.mediawiki.org/wiki/Help:Tables
+        String input = "{|\n"
+        +"! price\n"
+        +"! fruit\n"
+        +"! merchant\n"
+        +"|-\n"
+        +"| a || b <ref name=\"myref\">{{cite web|url=http://gnu.org|accessdate=2017-08-30}}</ref>  || c <ref name=\"ms\"> or {{cite journal|url=http://microsoft.com/|title=BLah}} </ref>\n"
+        +"|-\n"
+        +"| d || e <ref name=\"ms\"/>|| f <ref name=\"myref\" />\n"
+        +"|-\n"
+        +"|}\n";
+        
+        try {
+            prepareOptions(-1, true, true, null);
+            parse(input);
+        } catch (Exception e) {
+                Assert.fail("Parsing failed", e);
+        }
+        Assert.assertEquals(project.columnModel.columns.size(), 5);
+        Assert.assertEquals(project.rows.get(0).cells.get(1).value, "b");
+        Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://gnu.org");
+        Assert.assertEquals(project.rows.get(0).cells.get(4).value, "http://microsoft.com/");
+        Assert.assertEquals(project.rows.get(1).cells.get(4).value, "http://gnu.org");
+        Assert.assertEquals(project.rows.get(1).cells.get(2).value, "http://microsoft.com/");
+    }
+
     //--helpers--
     
     private void parse(String wikitext) {

main/webapp/modules/core/langs/translation-de.json

@@ -1,20 +1,109 @@
 {
     "name": "Deutsch",
     "core-index": {
+        "about": "Über",
+        "delete": "Löschen",
+        "slogan": "Ein leistungsstarkes Werkzeug für die Bearbeitung von unordentlichen Daten",
         "help": "Hilfe",
-        "version": "Version",
+        "new-proj-name": "Neuer Projektname:",
-        "new-version": "Neue Version!",
-        "download": "Herunterladen",
-        "now": "jetzt",
-        "preferences": "Einstellungen",
         "edit": "Ändern",
-        "delete": "Löschen"
+        "add-pref": "Einstellung hinzufügen",
+        "value": "Wert",
+        "no-proj": "Kein Projekt vorhanden. Wählen Sie auf der linken Seite \"Projekt erstellen\", um ein neues Projekt zu erstellen.",
+        "version": "Version",
+        "sample-data": "Beispieldatensätze",
+        "download": "Herunterladen",
+        "error-rename": "Fehler beim Umbenennen des Projekts:",
+        "new-version": "Neue Version!",
+        "now": "jetzt",
+        "preferences": "Einstellungen"
     },
     "core-index-create": {
+        "almost-done": "fast fertig…",
+        "min-remaining": "verbleibende Minuten",
+        "create-proj": "Projekt erstellen",
+        "memory-usage": "Speichernutzung:",
+        "question": "Erstellen Sie ein Projekt, um Daten zu importieren. Welche Art von Daten können Sie importieren?",
+        "sec-remaining": "Verbleibende Sekunden",
         "done": "Fertig.",
-        "almost-done": "fast fertig…"
+        "formats": "TSV, CSV, *SV, Excel (.xls und .xlsx), JSON, XML, RDF als XML, und Google Data Dokumente werden unterstützt. Die Unterstützung für andere Formate kann mit OpenRefine-Erweiterungen hinzugefügt werden.",
+        "no-details": "Keine technischen Details."
     },
     "core-index-import": {
-        "import-proj": "Projekt importieren"
+        "rename": "Projekt umbenennen (optional):",
+        "select-file": "Zu importierende Dateien auswählen",
+        "warning-name": "Bitte benennen Sie Ihr Projekt.",
+        "updating-preview": "Vorschau wird aktualisiert ...",
+        "import-proj": "Projekt importieren",
+        "warning-select": "Bitte wählen Sie mindestens eine Datei aus.",
+        "optional-separated": "Optional, kommagetrennt",
+        "file": "Projektdatei:",
+        "several-file": "Es sind mehrere Dateien verfügbar. Bitte wählen Sie die zu importierenden aus.",
+        "web-address": "Webadressen (URLs)",
+        "size": "Größe",
+        "errors": "Fehler:",
+        "column-widths": "Spaltenbreite:",
+        "clipboard": "Zwischenablage",
+        "import": "Importieren?",
+        "warning-web-address": "Sie müssen für den Import eine Webadresse (URL) angeben.",
+        "import-worksheet": "Zu importierende Arbeitsblätter",
+        "format": "Format",
+        "column-names": "Spaltenname:",
+        "uploading-data": "Daten werden hochgeladen ...",
+        "clipboard-label": "Daten aus der Zwischenablage hier einfügen:",
+        "this-computer": "Dieser Computer",
+        "name": "Name",
+        "mime-type": "MIME-Type",
+        "comma-separated": "Kommagetrennte Nummern",
+        "error": "Fehler:",
+        "downloading-data": "Daten werden heruntergeladen ...",
+        "unknown-err": "Unbekannter Fehler"
+    },
+    "core-index-open": {
+        "rename": "umbenennen",
+        "del-body": "Sind Sie sicher, dass Sie das Projekt \" löschen möchten",
+        "open-proj": "Projekt öffnen",
+        "warning-proj-name": "Sie müssen einen Projektnamen eingeben.",
+        "del-title": "Dieses Projekt löschen",
+        "warning-rename": "Fehler beim Umbenennen des Projekts:",
+        "new-title": "Neuer Projektname:",
+        "name": "Name"
+    },
+    "core-index-lang": {
+        "label": "Bevorzugte Sprache auswählen",
+        "send-req": "Sprache ändern",
+        "page-reload": "Die Seite wird aktualisiert, um die Änderung zu übernehmen.",
+        "lang-settings": "Spracheinstellungen"
+    },
+    "core-index-parser": {
+        "commas": "Kommas (CSV)",
+        "tabs": "Tabs (TSV)",
+        "invalid-wikitext": "Es konnte keine Tabelle geparst werden. Sind Sie sicher, dass dies eine gültige Wiki-Tabelle ist?",
+        "custom": "benutzerdefiniert",
+        "escape": "Makiere Sonderzeichen mit \\",
+        "click-xml": "Klicken Sie auf das erste XML-Element, das dem ersten zu ladenden Datensatz entspricht.",
+        "json-parser": "Klicken Sie auf den ersten JSON {} Knoten, der dem ersten zu ladenden Datensatz entspricht."
+    },
+    "core-dialogs": {
+        "cluster-size": "Clustergröße",
+        "find-more": "Mehr herausfinden ...",
+        "no-cluster-found": "Mit der ausgewählten Methode wurden keine Cluster gefunden",
+        "row-count": "Reihenanzahl",
+        "choice-avg-length": "Durchschnittliche Länge der Auswahl",
+        "fingerprint": "Fingerabdruck",
+        "reorder-column": "Spalten neu ordnen / entfernen",
+        "cluster-values": "Werte im Cluster",
+        "nearest-neighbor": "nächster Nachbar",
+        "try-another-method": "Versuchen Sie eine andere Methode oben auszuwählen oder ändern Sie ihre Parameter",
+        "new-cell-val": "Neuer Zellenwert",
+        "choice-var-length": "Längenabweichung der Auswahl",
+        "use-this-val": "Diesen Wert verwenden",
+        "found": "gefunden",
+        "template-suffix": "Suffix"
+    },
+    "core-buttons": {
+        "new-topic": "Neues Thema",
+        "copy": "Kopieren",
+        "transpose": "Transponieren"
     }
 }

main/webapp/modules/core/langs/translation-en.json

@@ -384,11 +384,11 @@
     "core-views": {
         "edit-cell": "Edit this cell",
         "choose-match": "Choose new match",
-        "match-all-cells": "Match this topic to this and all identical cells",
+        "match-all-cells": "Match this item to this and all identical cells",
-        "match-this-cell": "Match this topic to this cell",
+        "match-this-cell": "Match this item to this cell",
-        "create-topic-cells": "Create a new topic for this and all identical cells",
+        "create-topic-cells": "Create a new item for this and all identical cells",
-        "create-topic-cell": "Create a new topic for this cell",
+        "create-topic-cell": "Create a new item for this cell",
-        "create-topic": "Create new topic",
+        "create-topic": "Create new item",
         "search-match": "Search for match",
         "not-valid-number": "Not a valid number.",
         "not-valid-date": "Not a valid date.",
@@ -411,7 +411,7 @@
         "collapse-left": "Collapse all columns to left",
         "collapse-right": "Collapse all columns to right",
         "reconcile": "Reconcile",
-        "match-fb": "Match this column's cells to topics on Freebase",
+        "reconcile-tooltip": "Match this column's cells to items from external databases",
         "reverse": "Reverse",
         "remove-sort": "Remove sort",
         "sort-by": "Sort by",
@@ -445,7 +445,7 @@
         "warning-other-col": "Please select some other column to copy to.",
         "warning-sel-judg": "Please select at least one kind of judgment to copy.",
         "start-recon": "Start reconciling",
-        "recon-text-fb": "Reconcile text in this column with topics on Freebase",
+        "recon-text-fb": "Reconcile text in this column with items on Freebase",
         "facets": "Facets",
         "by-judg": "By judgment",
         "best-score": "Best candidate's score",
@@ -469,12 +469,12 @@
         "actions": "Actions",
         "best-cand": "Match each cell to its best candidate",
         "best-cand2": "Match each cell to its best candidate in this column for all current filtered rows",
-        "new-topic": "Create a new topic for each cell",
+        "new-topic": "Create a new item for each cell",
-        "new-topic2": "Mark to create one new topic for each cell in this column for all current filtered rows",
+        "new-topic2": "Mark to create one new item for each cell in this column for all current filtered rows",
-        "one-topic": "Create one new topic for similar cells",
+        "one-topic": "Create one new item for similar cells",
-        "one-topic2": "Mark to create one new topic for each group of similar cells in this column for all current filtered rows",
+        "one-topic2": "Mark to create one new item for each group of similar cells in this column for all current filtered rows",
         "filtered-cell": "Match all filtered cells to...",
-        "filtered-cell2": "Search for a topic to match all filtered cells to",
+        "filtered-cell2": "Search for a item to match all filtered cells to",
         "discard-judg": "Discard reconciliation judgments",
         "discard-judg2": "Discard reconciliation judgments in this column for all current filtered rows",
         "clear-recon": "Clear reconciliation data",
@@ -667,7 +667,7 @@
         "start-recon": "Start Reconciling",
         "add-service": "Add Service",
         "dont-reconcile": "Don't Reconcile Cell",
-        "new-topic": "New Topic",
+        "new-topic": "New Item",
         "match": "Match",
         "copy": "Copy",
         "transpose": "Transpose",

main/webapp/modules/core/langs/translation-es.json

@@ -406,7 +406,6 @@
 		"collapse-left": "Contraer todas las columnas a la izquierda",
 		"collapse-right": "Contraer todas las columnas a la derecha",
 		"reconcile": "Cotejar",
-		"match-fb": "Coincidir las celdas de esta columna con valores de Freebase",
 		"reverse": "Invertir",
 		"remove-sort": "Quitar orden",
 		"sort-by": "Ordenar por",

main/webapp/modules/core/langs/translation-he.json

@@ -416,7 +416,6 @@
 		"collapse-left": "כיווץ כל הטורים שמשמאל",
 		"collapse-right": "כיווץ כל הטורים מימין",
 		"reconcile": "השלמה",
-		"match-fb": "התאמת הטורים בטור זה לנושאים מתוך Freebase",
 		"reverse": "היפוך",
 		"remove-sort": "הסרת המיון",
 		"sort-by": "מיון לפי",

main/webapp/modules/core/langs/translation-it.json

@@ -406,7 +406,6 @@
         "collapse-left": "Collassa tutte le colonne a sinistra",
         "collapse-right": "Collassa tutte le colonne a destra",
         "reconcile": "Riconcilia",
-        "match-fb": "Matcha le celle di questa colonna con topic di Freebase",
         "reverse": "Inverti",
         "remove-sort": "Rimuovi ordinamento",
         "sort-by": "Ordina per",

main/webapp/modules/core/langs/translation-jp.json

@@ -406,7 +406,6 @@
 		"collapse-left": "左側すべてをたたむ",
 		"collapse-right": "右側すべてをたたむ",
 		"reconcile": "名寄せ",
-		"match-fb": "Freebaseでこのカラムのセルをトピックにマッチ",
 		"reverse": "反転",
 		"remove-sort": "ソートを削除",
 		"sort-by": "ソート",

main/webapp/modules/core/langs/translation-nb_NO.json

@@ -0,0 +1 @@
+{}

main/webapp/modules/core/langs/translation-ru.json

@@ -453,7 +453,6 @@
         "collapse-left": "Свернуть все колонки слева",
         "collapse-right": "Свернуть все колонки справа",
         "reconcile": "Сверка",
-        "match-fb": "Сверить данные этой колонки с информацией в Базе знаний",
         "reverse": "В обратном порядке",
         "remove-sort": "Убрать сортировку",
         "sort-by": "Сортировка",

main/webapp/modules/core/langs/translation-zh.json

@@ -406,7 +406,6 @@
         "collapse-left": "收起左侧列",
         "collapse-right": "收起右侧列",
         "reconcile": "搭配",
-        "match-fb": "将此列的单元格与Freebase中的主题搭配",
         "reverse": "反转",
         "remove-sort": "不排序",
         "sort-by": "排序，按照",

main/webapp/modules/core/scripts/views/data-table/cell-recon-preview-popup-header.html

@@ -1,5 +1,5 @@
 <div class="data-table-topic-popup-header">
-  <button class="button" title="Match topic to this cell" bind="matchButton"></button>
+  <button class="button" title="Match item to this cell" bind="matchButton"></button>
-  <button class="button" title="Match topic to all visible cells with same content" bind="matchSimilarButton"></button>
+  <button class="button" title="Match item to all visible cells with same content" bind="matchSimilarButton"></button>
   <button class="button" title="Cancel" bind="cancelButton"></button>
 </div>

main/webapp/modules/core/scripts/views/data-table/column-header-ui.js

@@ -187,7 +187,7 @@ DataTableColumnHeaderUI.prototype._createMenuForColumnHeader = function(elmt) {
     {
       id: "core/reconcile",
       label: $.i18n._('core-views')["reconcile"],
-      tooltip: $.i18n._('core-views')["match-fb"],
+      tooltip: $.i18n._('core-views')["reconcile-tooltip"],
       width: "170px",
       submenu: []
     }

main/webapp/modules/core/scripts/views/data-table/menu-reconcile.js

@@ -385,11 +385,10 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
             doReconMarkNewTopics(false);
           }
         },
-        {},
         {
           id: "core/match-similar-to-new-topic",
-          label: $.i18n._('core-views')["new-topic"],
+          label: $.i18n._('core-views')["one-topic"],
-          tooltip: $.i18n._('core-views')["new-topic2"],
+          tooltip: $.i18n._('core-views')["one-topic2"],
           click: function() {
             doReconMarkNewTopics(true);
           }