From a19267411827936e278c3d979b4ec5367da2da94 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Mon, 26 Jul 2010 22:50:00 +0000 Subject: [PATCH] - added smartSplit GEL function that can handle quoted values - added max width to operation extract dialog - made GEL get and slice functions handle HasFieldsList - fixed versioned standard-reconcile URLs (they need userid.user.dev) git-svn-id: http://google-refine.googlecode.com/svn/trunk@1110 7d457c2a-affb-35e4-300a-418c747d4874 --- .../metaweb/gridworks/expr/HasFieldsList.java | 4 + .../gridworks/expr/HasFieldsListImpl.java | 7 ++ .../metaweb/gridworks/expr/functions/Get.java | 33 +++++--- .../gridworks/expr/functions/Slice.java | 16 +++- .../expr/functions/strings/SmartSplit.java | 78 +++++++++++++++++++ .../gel/ControlFunctionRegistry.java | 2 + .../scripts/reconciliation/recon-dialog.js | 2 +- .../scripts/reconciliation/recon-manager.js | 2 +- .../modules/core/styles/widgets/history.css | 1 + 9 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 main/src/com/metaweb/gridworks/expr/functions/strings/SmartSplit.java diff --git a/main/src/com/metaweb/gridworks/expr/HasFieldsList.java b/main/src/com/metaweb/gridworks/expr/HasFieldsList.java index 82f85ff82..fafa6dc0f 100644 --- a/main/src/com/metaweb/gridworks/expr/HasFieldsList.java +++ b/main/src/com/metaweb/gridworks/expr/HasFieldsList.java @@ -7,4 +7,8 @@ package com.metaweb.gridworks.expr; */ public interface HasFieldsList extends HasFields { public int length(); + + public HasFields get(int index); + + public HasFieldsList getSubList(int from, int to); } diff --git a/main/src/com/metaweb/gridworks/expr/HasFieldsListImpl.java b/main/src/com/metaweb/gridworks/expr/HasFieldsListImpl.java index 5d341a049..357178854 100644 --- a/main/src/com/metaweb/gridworks/expr/HasFieldsListImpl.java +++ b/main/src/com/metaweb/gridworks/expr/HasFieldsListImpl.java @@ -31,4 +31,11 @@ public class HasFieldsListImpl extends ArrayList implements HasFields int c = size(); return (c > 0 && get(0).fieldAlsoHasFields(name)); } + + public HasFieldsList getSubList(int fromIndex, int toIndex) { + HasFieldsListImpl subList = new HasFieldsListImpl(); + subList.addAll(this.subList(fromIndex, toIndex)); + + return subList; + } } diff --git a/main/src/com/metaweb/gridworks/expr/functions/Get.java b/main/src/com/metaweb/gridworks/expr/functions/Get.java index 0f9d27e44..ed10b00c1 100644 --- a/main/src/com/metaweb/gridworks/expr/functions/Get.java +++ b/main/src/com/metaweb/gridworks/expr/functions/Get.java @@ -8,6 +8,7 @@ import org.json.JSONWriter; import com.metaweb.gridworks.expr.ExpressionUtils; import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.expr.HasFieldsList; import com.metaweb.gridworks.gel.Function; public class Get implements Function { @@ -19,16 +20,19 @@ public class Get implements Function { Object to = (args.length == 3) ? args[2] : null; if (v != null && from != null) { - if (v instanceof HasFields) { - if (from instanceof String) { - return ((HasFields) v).getField((String) from, bindings); - } + if (v instanceof HasFields && from instanceof String) { + return ((HasFields) v).getField((String) from, bindings); } else { if (from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); + if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList) { + int length = 0; + if (v.getClass().isArray()) { + length = ((Object[]) v).length; + } else if (v instanceof HasFieldsList) { + length = ((HasFieldsList) v).length(); + } else { + length = ExpressionUtils.toObjectList(v).size(); + } int start = ((Number) from).intValue(); if (start < 0) { @@ -37,10 +41,13 @@ public class Get implements Function { start = Math.min(length, Math.max(0, start)); if (to == null) { - return start >= length ? null : - (v.getClass().isArray() ? - ((Object[]) v)[start] : - ExpressionUtils.toObjectList(v).get(start)); + if (v.getClass().isArray()) { + return ((Object[]) v)[start]; + } else if (v instanceof HasFieldsList) { + return ((HasFieldsList) v).get(start); + } else { + return ExpressionUtils.toObjectList(v).get(start); + } } else { int end = (to != null) ? ((Number) to).intValue() : length; @@ -56,6 +63,8 @@ public class Get implements Function { System.arraycopy((Object[]) v, start, a2, 0, end - start); return a2; + } else if (v instanceof HasFieldsList) { + return ((HasFieldsList) v).getSubList(start, end); } else { return ExpressionUtils.toObjectList(v).subList(start, end); } diff --git a/main/src/com/metaweb/gridworks/expr/functions/Slice.java b/main/src/com/metaweb/gridworks/expr/functions/Slice.java index 2e233eaa4..cfc0d0ebb 100644 --- a/main/src/com/metaweb/gridworks/expr/functions/Slice.java +++ b/main/src/com/metaweb/gridworks/expr/functions/Slice.java @@ -7,6 +7,7 @@ import org.json.JSONException; import org.json.JSONWriter; import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.HasFieldsList; import com.metaweb.gridworks.gel.Function; public class Slice implements Function { @@ -18,10 +19,15 @@ public class Slice implements Function { Object to = (args.length == 3) ? args[2] : null; if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); + if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList) { + int length = 0; + if (v.getClass().isArray()) { + length = ((Object[]) v).length; + } else if (v instanceof HasFieldsList) { + length = ((HasFieldsList) v).length(); + } else { + length = ExpressionUtils.toObjectList(v).size(); + } int start = ((Number) from).intValue(); int end = (to != null) ? ((Number) to).intValue() : length; @@ -42,6 +48,8 @@ public class Slice implements Function { System.arraycopy((Object[]) v, start, a2, 0, end - start); return a2; + } else if (v instanceof HasFieldsList) { + return ((HasFieldsList) v).getSubList(start, end); } else { return ExpressionUtils.toObjectList(v).subList(start, end); } diff --git a/main/src/com/metaweb/gridworks/expr/functions/strings/SmartSplit.java b/main/src/com/metaweb/gridworks/expr/functions/strings/SmartSplit.java new file mode 100644 index 000000000..c0d6bf41c --- /dev/null +++ b/main/src/com/metaweb/gridworks/expr/functions/strings/SmartSplit.java @@ -0,0 +1,78 @@ +package com.metaweb.gridworks.expr.functions.strings; + +import java.io.IOException; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import au.com.bytecode.opencsv.CSVParser; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class SmartSplit implements Function { + static protected CSVParser s_tabParser = new CSVParser( + '\t', + CSVParser.DEFAULT_QUOTE_CHARACTER, + CSVParser.DEFAULT_ESCAPE_CHARACTER, + CSVParser.DEFAULT_STRICT_QUOTES, + CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, + false + ); + static protected CSVParser s_commaParser = new CSVParser( + ',', + CSVParser.DEFAULT_QUOTE_CHARACTER, + CSVParser.DEFAULT_ESCAPE_CHARACTER, + CSVParser.DEFAULT_STRICT_QUOTES, + CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, + false + ); + public Object call(Properties bindings, Object[] args) { + if (args.length >= 1 && args.length <= 2) { + CSVParser parser = null; + + Object v = args[0]; + String s = v.toString(); + + if (args.length > 1) { + String sep = args[1].toString(); + parser = new CSVParser( + sep.charAt(0), + CSVParser.DEFAULT_QUOTE_CHARACTER, + CSVParser.DEFAULT_ESCAPE_CHARACTER, + CSVParser.DEFAULT_STRICT_QUOTES, + CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, + false + ); + } + + if (parser == null) { + int tab = s.indexOf('\t'); + if (tab >= 0) { + parser = s_tabParser; + } else { + parser = s_commaParser; + } + } + + try { + return parser.parseLine(s); + } catch (IOException e) { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " error: " + e.getMessage()); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 or 2 strings"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the array of strings obtained by splitting s with separator sep. Handles quotes properly. Guesses tab or comma separator if \"sep\" is not given."); + writer.key("params"); writer.value("string s, optional string sep"); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/main/src/com/metaweb/gridworks/gel/ControlFunctionRegistry.java b/main/src/com/metaweb/gridworks/gel/ControlFunctionRegistry.java index 29a5e9516..fb9d3c120 100644 --- a/main/src/com/metaweb/gridworks/gel/ControlFunctionRegistry.java +++ b/main/src/com/metaweb/gridworks/gel/ControlFunctionRegistry.java @@ -53,6 +53,7 @@ import com.metaweb.gridworks.expr.functions.strings.Reinterpret; import com.metaweb.gridworks.expr.functions.strings.Replace; import com.metaweb.gridworks.expr.functions.strings.ReplaceChars; import com.metaweb.gridworks.expr.functions.strings.SHA1; +import com.metaweb.gridworks.expr.functions.strings.SmartSplit; import com.metaweb.gridworks.expr.functions.strings.Split; import com.metaweb.gridworks.expr.functions.strings.SplitByCharType; import com.metaweb.gridworks.expr.functions.strings.SplitByLengths; @@ -133,6 +134,7 @@ public class ControlFunctionRegistry { registerFunction("replace", new Replace()); registerFunction("replaceChars", new ReplaceChars()); registerFunction("split", new Split()); + registerFunction("smartSplit", new SmartSplit()); registerFunction("splitByCharType", new SplitByCharType()); registerFunction("splitByLengths", new SplitByLengths()); registerFunction("partition", new Partition()); diff --git a/main/webapp/modules/core/scripts/reconciliation/recon-dialog.js b/main/webapp/modules/core/scripts/reconciliation/recon-dialog.js index 3c0c8b08e..63c54434c 100644 --- a/main/webapp/modules/core/scripts/reconciliation/recon-dialog.js +++ b/main/webapp/modules/core/scripts/reconciliation/recon-dialog.js @@ -181,7 +181,7 @@ ReconDialog.prototype._onAddNamespacedService = function() { var namespaceData = elmts.namespaceInput.data("data.suggest"); var typeData = elmts.typeInput.data("data.suggest"); if (namespaceData) { - var url = "http://1.standard-reconcile.freebaseapps.com/namespace_reconcile?namespace=" + + var url = "http://1.standard-reconcile.dfhuynh.user.dev.freebaseapps.com/namespace_reconcile?namespace=" + escape(namespaceData.id); if (typeData) { url += "&type=" + typeData.id; diff --git a/main/webapp/modules/core/scripts/reconciliation/recon-manager.js b/main/webapp/modules/core/scripts/reconciliation/recon-manager.js index 49d1f7b5b..de3cc5215 100644 --- a/main/webapp/modules/core/scripts/reconciliation/recon-manager.js +++ b/main/webapp/modules/core/scripts/reconciliation/recon-manager.js @@ -109,7 +109,7 @@ ReconciliationManager.save = function(f) { ReconciliationManager._rebuildMap(); } else { ReconciliationManager.registerStandardService( - "http://1.standard-reconcile.freebaseapps.com/reconcile"); + "http://1.standard-reconcile.dfhuynh.user.dev.freebaseapps.com/reconcile"); } }, dataType: "json" diff --git a/main/webapp/modules/core/styles/widgets/history.css b/main/webapp/modules/core/styles/widgets/history.css index cec9dc5e4..a400d8ed4 100644 --- a/main/webapp/modules/core/styles/widgets/history.css +++ b/main/webapp/modules/core/styles/widgets/history.css @@ -81,6 +81,7 @@ a.history-entry:hover { height: 400px; padding: 2px; border: 1px inset; + max-width: 400px !important; overflow: auto; } textarea.history-operation-json {