From ebdc40ad7144d299187c486cb0fe660188b41d56 Mon Sep 17 00:00:00 2001 From: Frank Wennerdahl Date: Mon, 21 Jan 2013 08:21:16 +0100 Subject: [PATCH 01/53] Added CSV quote options Added two additional CSV options, one for parsing and one for export. Specifying strict quotes when parsing will ignore all data not quoted. Specifying quote all when exporting will enclose all values in quotes. No front-end changes made, just added the support for the options in the requests. --- main/src/com/google/refine/exporters/CsvExporter.java | 3 ++- .../com/google/refine/importers/SeparatorBasedImporter.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/main/src/com/google/refine/exporters/CsvExporter.java b/main/src/com/google/refine/exporters/CsvExporter.java index 62834f963..3766539b1 100644 --- a/main/src/com/google/refine/exporters/CsvExporter.java +++ b/main/src/com/google/refine/exporters/CsvExporter.java @@ -81,6 +81,7 @@ public class CsvExporter implements WriterExporter{ JSONUtilities.getString(options, "separator", Character.toString(this.separator)); final String lineSeparator = options == null ? CSVWriter.DEFAULT_LINE_END : JSONUtilities.getString(options, "lineSeparator", CSVWriter.DEFAULT_LINE_END); + final boolean quoteAll = options == null ? false : JSONUtilities.getBoolean(options, "quoteAll", false); final boolean printColumnHeader = (params != null && params.getProperty("printColumnHeader") != null) ? @@ -110,7 +111,7 @@ public class CsvExporter implements WriterExporter{ cellData.text : ""; } - csvWriter.writeNext(strings, false); + csvWriter.writeNext(strings, quoteAll); } } }; diff --git a/main/src/com/google/refine/importers/SeparatorBasedImporter.java b/main/src/com/google/refine/importers/SeparatorBasedImporter.java index cd28d92bf..b70cda4d2 100644 --- a/main/src/com/google/refine/importers/SeparatorBasedImporter.java +++ b/main/src/com/google/refine/importers/SeparatorBasedImporter.java @@ -95,12 +95,13 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { } sep = StringEscapeUtils.unescapeJava(sep); boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true); + boolean strictQuotes = JSONUtilities.getBoolean(options, "strictQuotes", false); final CSVParser parser = new CSVParser( sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators. CSVParser.DEFAULT_QUOTE_CHARACTER, (char) 0, // escape character - CSVParser.DEFAULT_STRICT_QUOTES, + strictQuotes, CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, !processQuotes); From 1f7ab046c75bfacc6adee46f74c3b8d7feff9613 Mon Sep 17 00:00:00 2001 From: Frank Wennerdahl Date: Thu, 24 Jan 2013 09:04:17 +0100 Subject: [PATCH 02/53] Quotes should not be removed from values Leading quotation marks should not be removed from values. If they have been left by the importing parser they should be considered part of the value. --- main/src/com/google/refine/importers/ImporterUtilities.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main/src/com/google/refine/importers/ImporterUtilities.java b/main/src/com/google/refine/importers/ImporterUtilities.java index 47f94756b..fe2ea3001 100644 --- a/main/src/com/google/refine/importers/ImporterUtilities.java +++ b/main/src/com/google/refine/importers/ImporterUtilities.java @@ -57,10 +57,6 @@ public class ImporterUtilities { static public Serializable parseCellValue(String text) { if (text.length() > 0) { - if (text.length() > 1 && text.startsWith("\"") && text.endsWith("\"")) { - return text.substring(1, text.length() - 1); - } - String text2 = text.trim(); if (text2.length() > 0) { try { From 7b3379afc70d2836b294c7865c870e3c0ab3fe47 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Tue, 26 Feb 2013 16:35:21 -0500 Subject: [PATCH 03/53] fix range check in getFields - fixes issue 687 --- main/src/com/google/refine/model/Recon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/src/com/google/refine/model/Recon.java b/main/src/com/google/refine/model/Recon.java index fac984aaf..43dd7e0a0 100644 --- a/main/src/com/google/refine/model/Recon.java +++ b/main/src/com/google/refine/model/Recon.java @@ -253,7 +253,7 @@ public class Recon implements HasFields, Jsonizable { @Override public Object getField(String name, Properties bindings) { int index = s_featureMap.containsKey(name) ? s_featureMap.get(name) : -1; - return (index > 0 && index < features.length) ? features[index] : null; + return (index >= 0 && index < features.length) ? features[index] : null; } @Override From 38a88db58d8c4768e5b38e87878cd96e4d77aacb Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Fri, 1 Mar 2013 18:57:11 -0500 Subject: [PATCH 04/53] Patch Suggest to accept type param for Search --- main/webapp/modules/core/externals/suggest/suggest-4_1.js | 2 +- main/webapp/modules/core/externals/suggest/suggest-4_1.min.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/webapp/modules/core/externals/suggest/suggest-4_1.js b/main/webapp/modules/core/externals/suggest/suggest-4_1.js index a93a44eb8..ad1547b5d 100644 --- a/main/webapp/modules/core/externals/suggest/suggest-4_1.js +++ b/main/webapp/modules/core/externals/suggest/suggest-4_1.js @@ -87,7 +87,7 @@ var SEARCH_PARAMS = { key:1, filter:1, spell:1, exact:1, lang:1, scoring:1, prefixed:1, stemmed:1, format:1, mql_output:1, - output:1 + output:1, type:1 }; $.suggest = function(name, prototype) { diff --git a/main/webapp/modules/core/externals/suggest/suggest-4_1.min.js b/main/webapp/modules/core/externals/suggest/suggest-4_1.min.js index b5d275589..4ef28c2dd 100644 --- a/main/webapp/modules/core/externals/suggest/suggest-4_1.min.js +++ b/main/webapp/modules/core/externals/suggest/suggest-4_1.min.js @@ -34,7 +34,7 @@ * */ (function(c,q){if(!("console"in window)){var o=window.console={};o.log=o.warn=o.error=o.debug=function(){}}c(function(){var a=c("
");c(document.body).append(a);var b=setTimeout(function(){if(c.cleanData){var a=c.cleanData;c.cleanData=function(b){for(var e=0,g;null!=(g=b[e]);e++)c(g).triggerHandler("remove");a(b)}}else{var b=c.fn.remove;c.fn.remove=function(a,f){return this.each(function(){f||(!a||c.filter(a,[this]).length)&&c("*",this).add([this]).each(function(){c(this).triggerHandler("remove")}); -return b.call(c(this),a,f)})}}},1);a.bind("remove",function(){clearTimeout(b)});a.remove()});var p={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1};c.suggest=function(a,b){c.fn[a]=function(b){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+ +return b.call(c(this),a,f)})}}},1);a.bind("remove",function(){clearTimeout(b)});a.remove()});var p={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1,type:1};c.suggest=function(a,b){c.fn[a]=function(b){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+ this.nodeName));var g=c.data(this,a);g&&g._destroy();c.data(this,a,new c.suggest[a](this,b))._init()})};c.suggest[a]=function(b,g){var d=this,e=this.options=c.extend(!0,{},c.suggest.defaults,c.suggest[a].defaults,g),j=e.css_prefix=e.css_prefix||"",h=e.css;this.name=a;c.each(h,function(a){h[a]=j+h[a]});e.ac_param={};c.each(p,function(a){var b=e[a];null===b||""===b||(e.ac_param[a]=b)});e.flyout_lang=null;if(e.ac_param.lang){var i=e.ac_param.lang;"string"===c.type(i)&&(i=i.split(","));if(c.isArray(i)&& i.length&&(i=c.trim(i[0])))e.flyout_lang=i}this._status={START:"",LOADING:"",SELECT:"",ERROR:""};e.status&&(e.status instanceof Array&&3<=e.status.length)&&(this._status.START=e.status[0]||"",this._status.LOADING=e.status[1]||"",this._status.SELECT=e.status[2]||"",4===e.status.length&&(this._status.ERROR=e.status[3]||""));var i=this.status=c('
').addClass(h.status),k=this.list=c("