s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 475 - Support escaped custom separators

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2355 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2011-11-04 19:04:16 +00:00
parent cacbedd352
commit a7c81880a8
7 changed files with 36 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
main/src/com/google/refine/importers/FixedWidthImporter.java
View File

@ -44,7 +44,7 @@ public class FixedWidthImporter extends TabularImportingParserBase {
}
}
JSONUtilities.safePut(options, "lineSeparator", "\n");
JSONUtilities.safePut(options, "lineSeparator", "\\n");
JSONUtilities.safePut(options, "headerLines", 0);
JSONUtilities.safePut(options, "columnWidths", columnWidths);
JSONUtilities.safePut(options, "guessCellValueTypes", true);
@ -63,7 +63,8 @@ public class FixedWidthImporter extends TabularImportingParserBase {
JSONObject options,
List<Exception> exceptions
) {
// String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\n");
// String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\\n");
// lineSeparator = StringEscapeUtils.unescapeJava(lineSeparator);
final int[] columnWidths = JSONUtilities.getIntArray(options, "columnWidths");
List<Object> retrievedColumnNames = null;

2
main/src/com/google/refine/importers/LineBasedImporter.java
View File

@ -27,7 +27,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
ImportingJob job, List<JSONObject> fileRecords, String format) {
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
JSONUtilities.safePut(options, "lineSeparator", "\n");
JSONUtilities.safePut(options, "lineSeparator", "\\n");
JSONUtilities.safePut(options, "linesPerRow", 1);
JSONUtilities.safePut(options, "headerLines", 0);
JSONUtilities.safePut(options, "guessCellValueTypes", true);

15
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -48,6 +48,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringEscapeUtils;
import org.json.JSONObject;
import au.com.bytecode.opencsv.CSVParser;
@ -68,10 +69,10 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
List<JSONObject> fileRecords, String format) {
JSONObject options = super.createParserUIInitializationData(job, fileRecords, format);
JSONUtilities.safePut(options, "lineSeparator", "\n");
JSONUtilities.safePut(options, "lineSeparator", "\\n");
String separator = guessSeparator(job, fileRecords);
JSONUtilities.safePut(options, "separator", separator != null ? separator : "\t");
JSONUtilities.safePut(options, "separator", separator != null ? separator : "\\t");
JSONUtilities.safePut(options, "guessCellValueTypes", true);
JSONUtilities.safePut(options, "processQuotes", true);
@ -90,11 +91,13 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
JSONObject options,
List<Exception> exceptions
) {
// String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\n");
String sep = JSONUtilities.getString(options, "separator", "\t");
// String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\\n");
// lineSeparator = StringEscapeUtils.unescapeJava(lineSeparator);
String sep = JSONUtilities.getString(options, "separator", "\\t");
if (sep == null || "".equals(sep)) {
sep = "\t";
sep = "\\t";
}
sep = StringEscapeUtils.unescapeJava(sep);
boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true);
final CSVParser parser = new CSVParser(
@ -149,7 +152,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
File file = new File(job.getRawDataDir(), location);
Separator separator = guessSeparator(file, encoding);
if (separator != null) {
return Character.toString(separator.separator);
return StringEscapeUtils.escapeJava(Character.toString(separator.separator));
}
}
}

14
main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js
View File

@ -34,8 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
function CustomTabularExporterDialog(options) {
options = options || {
format: 'tsv',
lineSeparator: '\n',
separator: '\t',
lineSeparator: '\\n',
separator: '\\t',
encoding: 'UTF-8',
outputColumnHeaders: true,
outputBlankRows: false,
@ -202,8 +202,8 @@ CustomTabularExporterDialog.prototype._createDialog = function(options) {
CustomTabularExporterDialog.prototype._configureUIFromOptionCode = function(options) {
this._dialog.find('input[name="custom-tabular-exporter-download-format"][value="' + options.format + '"]').attr('checked', 'checked');
this._elmts.separatorInput[0].value = String.encodeSeparator(options.separator || ',');
this._elmts.lineSeparatorInput[0].value = String.encodeSeparator(options.lineSeparator || '\n');
this._elmts.separatorInput[0].value = options.separator || ',';
this._elmts.lineSeparatorInput[0].value = options.lineSeparator || '\\n';
this._elmts.encodingInput[0].value = options.encoding;
this._elmts.outputColumnHeadersCheckbox.attr('checked', (options.outputColumnHeaders) ? 'checked' : '');
this._elmts.outputBlankRowsCheckbox.attr('checked', (options.outputBlankRows) ? 'checked' : '');
@ -336,13 +336,13 @@ CustomTabularExporterDialog.prototype._getOptionCode = function() {
if (options.format == 'tsv' || options.format == 'csv' || options.format == '*sv') {
if (options.format == 'tsv') {
options.separator = '\t';
options.separator = '\\t';
} else if (options.format == 'csv') {
options.separator = ',';
} else {
options.separator = String.decodeSeparator(this._elmts.separatorInput.val());
options.separator = this._elmts.separatorInput.val();
}
options.lineSeparator = String.decodeSeparator(this._elmts.lineSeparatorInput.val());
options.lineSeparator = this._elmts.lineSeparatorInput.val();
options.encoding = this._elmts.encodingInput.val();
}
options.outputColumnHeaders = this._elmts.outputColumnHeadersCheckbox[0].checked;

9
main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.js
View File

@ -73,9 +73,9 @@ Refine.FixedWidthParserUI.prototype.getOptions = function() {
}
if (this._optionContainer.find("input[name='row-separator']:checked")[0].value === "new-line") {
options.lineSeparator = "\n";
options.lineSeparator = "\\n";
} else {
options.lineSeparator = String.decodeSeparator(this._optionContainerElmts.rowSeparatorInput[0].value);
options.lineSeparator = this._optionContainerElmts.rowSeparatorInput[0].value;
}
var parseIntDefault = function(s, def) {
@ -140,11 +140,10 @@ Refine.FixedWidthParserUI.prototype._initialize = function() {
this._optionContainerElmts.columnNamesInput[0].value = this._config.columnNames.join(',');
}
var rowSeparatorValue = (this._config.lineSeparator == "\n") ? 'new-line' : 'custom';
var rowSeparatorValue = (this._config.lineSeparator == "\\n") ? 'new-line' : 'custom';
this._optionContainer.find(
"input[name='row-separator'][value='" + rowSeparatorValue + "']").attr("checked", "checked");
this._optionContainerElmts.rowSeparatorInput[0].value =
String.encodeSeparator(this._config.lineSeparator);
this._optionContainerElmts.rowSeparatorInput[0].value = this._config.lineSeparator;
if (this._config.ignoreLines > 0) {
this._optionContainerElmts.ignoreCheckbox.attr("checked", "checked");

9
main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.js
View File

@ -81,9 +81,9 @@ Refine.LineBasedParserUI.prototype.getOptions = function() {
options.linesPerRow = parseIntDefault(this._optionContainerElmts.linesPerRowInput[0].value, 1);
if (this._optionContainer.find("input[name='row-separator']:checked")[0].value === "new-line") {
options.lineSeparator = "\n";
options.lineSeparator = "\\n";
} else {
options.lineSeparator = String.decodeSeparator(this._optionContainerElmts.rowSeparatorInput[0].value);
options.lineSeparator = this._optionContainerElmts.rowSeparatorInput[0].value;
}
if (this._optionContainerElmts.ignoreCheckbox[0].checked) {
@ -127,11 +127,10 @@ Refine.LineBasedParserUI.prototype._initialize = function() {
this._optionContainerElmts.linesPerRowInput[0].value =
this._config.linesPerRow.toString();
var rowSeparatorValue = (this._config.lineSeparator == "\n") ? 'new-line' : 'custom';
var rowSeparatorValue = (this._config.lineSeparator == "\\n") ? 'new-line' : 'custom';
this._optionContainer.find(
"input[name='row-separator'][value='" + rowSeparatorValue + "']").attr("checked", "checked");
this._optionContainerElmts.rowSeparatorInput[0].value =
String.encodeSeparator(this._config.lineSeparator);
this._optionContainerElmts.rowSeparatorInput[0].value = this._config.lineSeparator;
if (this._config.ignoreLines > 0) {
this._optionContainerElmts.ignoreCheckbox.attr("checked", "checked");

19
main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.js
View File

@ -66,9 +66,9 @@ Refine.SeparatorBasedParserUI.prototype.getOptions = function() {
};
if (this._optionContainer.find("input[name='row-separator']:checked")[0].value === "new-line") {
options.lineSeparator = "\n";
options.lineSeparator = "\\n";
} else {
options.lineSeparator = String.decodeSeparator(this._optionContainerElmts.rowSeparatorInput[0].value);
options.lineSeparator = this._optionContainerElmts.rowSeparatorInput[0].value;
}
switch (this._optionContainer.find("input[name='column-separator']:checked")[0].value) {
@ -76,11 +76,10 @@ Refine.SeparatorBasedParserUI.prototype.getOptions = function() {
options.separator = ",";
break;
case 'tab':
options.separator = "\t";
options.separator = "\\t";
break;
default:
options.separator = String.decodeSeparator(
this._optionContainerElmts.columnSeparatorInput[0].value);
options.separator = this._optionContainerElmts.columnSeparatorInput[0].value;
}
var parseIntDefault = function(s, def) {
@ -141,18 +140,16 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
});
});
var rowSeparatorValue = (this._config.lineSeparator == "\n") ? 'new-line' : 'custom';
var rowSeparatorValue = (this._config.lineSeparator == "\\n") ? 'new-line' : 'custom';
this._optionContainer.find(
"input[name='row-separator'][value='" + rowSeparatorValue + "']").attr("checked", "checked");
this._optionContainerElmts.rowSeparatorInput[0].value =
String.encodeSeparator(this._config.lineSeparator);
this._optionContainerElmts.rowSeparatorInput[0].value = this._config.lineSeparator;
var columnSeparatorValue = (this._config.separator == ",") ? 'comma' :
((this._config.separator == "\t") ? 'tab' : 'custom');
((this._config.separator == "\\t") ? 'tab' : 'custom');
this._optionContainer.find(
"input[name='column-separator'][value='" + columnSeparatorValue + "']").attr("checked", "checked");
this._optionContainerElmts.columnSeparatorInput[0].value =
String.encodeSeparator(this._config.separator);
this._optionContainerElmts.columnSeparatorInput[0].value = this._config.separator;
if (this._config.ignoreLines > 0) {
this._optionContainerElmts.ignoreCheckbox.attr("checked", "checked");