s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1731 from xseris/master

Browse Source 
Custom column names for separator based importer
This commit is contained in:
Antonin Delpeuch 2018-09-24 08:29:46 +01:00 committed by GitHub
commit 22be1ec323
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 117 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -101,6 +101,28 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true);
boolean strictQuotes = JSONUtilities.getBoolean(options, "strictQuotes", false);
List<Object> retrievedColumnNames = null;
if (options.has("columnNames")) {
String[] strings = JSONUtilities.getStringArray(options, "columnNames");
if (strings.length > 0) {
retrievedColumnNames = new ArrayList<Object>();
for (String s : strings) {
s = s.trim();
if (!s.isEmpty()) {
retrievedColumnNames.add(s);
}
}
if (!retrievedColumnNames.isEmpty()) {
JSONUtilities.safePut(options, "headerLines", 1);
} else {
retrievedColumnNames = null;
}
}
}
final List<Object> columnNames = retrievedColumnNames;
Character quote = CSVParser.DEFAULT_QUOTE_CHARACTER;
String quoteCharacter = JSONUtilities.getString(options, "quoteCharacter", null);
if (quoteCharacter != null && quoteCharacter.trim().length() == 1) {
@ -118,14 +140,20 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
final LineNumberReader lnReader = new LineNumberReader(reader);
TableDataReader dataReader = new TableDataReader() {
boolean usedColumnNames = false;
@Override
public List<Object> getNextRowOfCells() throws IOException {
if (columnNames != null && !usedColumnNames) {
usedColumnNames = true;
return columnNames;
} else {
String line = lnReader.readLine();
if (line == null) {
return null;
} else {
return getCells(line, parser, lnReader);
}
}
}
};

49
main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java
View File

@ -38,6 +38,7 @@ import static org.mockito.Mockito.verify;
import java.io.StringReader;
import org.json.JSONArray;
import org.json.JSONException;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@ -514,6 +515,27 @@ public class TsvCsvImporterTests extends ImporterTest {
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
}
@Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readCustomColumnNames(String sep){
//create input
String inputSeparator = sep == null ? "\t" : sep;
String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3\n";
try {
prepareOptions(sep, -1, 0, 0, 1, false, false,"\"","[col1,col2,col3]");
parseOneFile(SUT, new StringReader(input));
} catch (Exception e) {
Assert.fail("Exception during file parse",e);
}
Assert.assertEquals(project.columnModel.columns.size(), 3);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "col1");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "col2");
Assert.assertEquals(project.columnModel.columns.get(2).getName(), "col3");
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data1");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data2");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
}
//---------------------read tests------------------------
@Test
public void readCsvWithProperties() {
@ -580,16 +602,24 @@ public class TsvCsvImporterTests extends ImporterTest {
String sep, int limit, int skip, int ignoreLines,
int headerLines, boolean guessValueType, boolean ignoreQuotes, String quoteCharacter) {
whenGetStringOption("separator", options, sep);
whenGetStringOption("quoteCharacter", options, quoteCharacter);
whenGetIntegerOption("limit", options, limit);
whenGetIntegerOption("skipDataLines", options, skip);
whenGetIntegerOption("ignoreLines", options, ignoreLines);
whenGetIntegerOption("headerLines", options, headerLines);
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
prepareOptions(sep, limit, skip, ignoreLines, headerLines, guessValueType, ignoreQuotes, quoteCharacter,"[]");
}
protected void prepareOptions(
String sep, int limit, int skip, int ignoreLines,
int headerLines, boolean guessValueType, boolean ignoreQuotes, String quoteCharacter, String columnNames) {
whenGetStringOption("separator", options, sep);
whenGetStringOption("quoteCharacter", options, quoteCharacter);
whenGetIntegerOption("limit", options, limit);
whenGetIntegerOption("skipDataLines", options, skip);
whenGetIntegerOption("ignoreLines", options, ignoreLines);
whenGetIntegerOption("headerLines", options, headerLines);
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
whenGetArrayOption("columnNames", options, new JSONArray(columnNames));
}
private void verifyOptions() {
try {
@ -601,6 +631,7 @@ public class TsvCsvImporterTests extends ImporterTest {
verify(options, times(1)).getBoolean("guessCellValueTypes");
verify(options, times(1)).getBoolean("processQuotes");
verify(options, times(1)).getBoolean("storeBlankCellsAsNulls");
verify(options, times(1)).getJSONArray("columnNames");
} catch (JSONException e) {
Assert.fail("JSON exception",e);
}

4
main/webapp/modules/core/langs/translation-en.json
View File

@ -155,7 +155,9 @@
"escape": "Escape special characters with \\",
"use-quote": "Use character",
"quote-delimits-cells": "to enclose cells containing column separators",
"click-xml": "Click on the first XML element corresponding to the first record to load."
"click-xml": "Click on the first XML element corresponding to the first record to load.",
"column-names-label": "Column names (comma separated)",
"column-names-optional":"comma separated"
},
"core-dialogs": {
"cluster-edit": "Cluster & Edit column",

13
main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.html
View File

@ -1,4 +1,4 @@
<div class="grid-layout layout-loose layout-full"><table>
<div class="grid-layout layout-loose layout-full"><table>
<tr>
<td><div class="grid-layout layout-tighter"><table>
<tr>
@ -24,7 +24,7 @@
<td><label for="$column-separator-custom" id="or-import-custom"></label>
<input bind="columnSeparatorInput" type="text" class="lightweight" size="5" /></td></tr>
<tr><td colspan="2" id="or-import-escape"></td></tr>
</table></div></td>
</table></div></td>
<td colspan="2"><div class="grid-layout layout-tightest"><table>
<tr><td width="1%"><input type="checkbox" bind="ignoreCheckbox" id="$ignore" /></td>
@ -55,9 +55,14 @@
</table></div></td>
</tr>
<tr>
<td>&nbsp;</td>
<td><div class="grid-layout layout-tightest" style="width:fit-content;"><table>
<tr><td width="1%"><input type="checkbox" bind="columnNamesCheckbox" id="$check-column-names" />
<label id="or-import-columnNames"></label></td></tr>
<tr>
<td><input style="width: 25em;" bind="columnNamesInput" /></td>
</tr></table></div></td>
<td><div class="grid-layout layout-tightest"><table>
<td colspan="1"><div class="grid-layout layout-tightest"><table>
<tr><td width="1%"><input type="checkbox" bind="guessCellValueTypesCheckbox" id="$guess" /></td>
<td><label for="$guess" id="or-import-parseCell"></label></td></tr>
</table></div></td>

37
main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.js
View File

@ -117,6 +117,13 @@ Refine.SeparatorBasedParserUI.prototype.getOptions = function() {
options.storeBlankCellsAsNulls = this._optionContainerElmts.storeBlankCellsAsNullsCheckbox[0].checked;
options.includeFileSources = this._optionContainerElmts.includeFileSourcesCheckbox[0].checked;
if (this._optionContainerElmts.columnNamesCheckbox[0].checked) {
var columnNames = this._optionContainerElmts.columnNamesInput.val();
if (columnNames != undefined && columnNames != null && columnNames != '') {
options.columnNames = columnNames.split(",");
}
}
return options;
};
@ -136,6 +143,10 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
$('#or-import-tabs').html($.i18n._('core-index-parser')["tabs"]);
$('#or-import-custom').html($.i18n._('core-index-parser')["custom"]);
$('#or-import-escape').html($.i18n._('core-index-parser')["escape"]);
$('#or-import-columnNames').html($.i18n._('core-index-parser')["column-names-label"] + ':');
$('#or-import-optional').html($.i18n._('core-index-parser')["column-names-optional"]);
self._optionContainerElmts.columnNamesInput.prop('disabled', true);
$('#or-import-ignore').text($.i18n._('core-index-parser')["ignore-first"]);
$('#or-import-lines').text($.i18n._('core-index-parser')["lines-beg"]);
@ -160,6 +171,31 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
});
});
this._optionContainerElmts.headerLinesCheckbox.on("click", function() {
if ($(this).is(':checked')) {
var isDisabled = $('textbox').prop('disabled');
if (!isDisabled) {
self._optionContainerElmts.columnNamesInput.prop('disabled', true);
self._optionContainerElmts.columnNamesCheckbox.prop("checked", false);
self._optionContainerElmts.columnNamesInput.val('');
}
} else {
self._optionContainerElmts.columnNamesInput.prop('disabled', false);
self._optionContainerElmts.columnNamesCheckbox.prop("checked", true);
}
});
this._optionContainerElmts.columnNamesCheckbox.on("click", function() {
if ($(this).is(':checked')) {
self._optionContainerElmts.headerLinesCheckbox.prop("checked", false);
self._optionContainerElmts.columnNamesInput.prop('disabled', false);
} else {
self._optionContainerElmts.headerLinesCheckbox.prop("checked", true);
self._optionContainerElmts.columnNamesInput.val('');
self._optionContainerElmts.columnNamesInput.prop('disabled', true);
}
});
var columnSeparatorValue = (this._config.separator == ",") ? 'comma' :
((this._config.separator == "\\t") ? 'tab' : 'custom');
this._optionContainer.find(
@ -206,6 +242,7 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
};
this._optionContainer.find("input").bind("change", onChange);
this._optionContainer.find("select").bind("change", onChange);
this._optionContainerElmts.columnNamesInput.bind("keyup",onChange);
};
Refine.SeparatorBasedParserUI.prototype._scheduleUpdatePreview = function() {