From 017a825600278b4eee05bdfe3ae8636b06534287 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Wed, 26 May 2010 05:42:31 +0000 Subject: [PATCH] Initial implementation of a templating exporter. git-svn-id: http://google-refine.googlecode.com/svn/trunk@860 7d457c2a-affb-35e4-300a-418c747d4874 --- CHANGES.txt | 5 +- .../commands/project/ExportRowsCommand.java | 22 ++- .../commands/row/GetRowsCommand.java | 8 +- .../exporters/TemplatingExporter.java | 101 ++++++++++ .../gridworks/expr/functions/Jsonize.java | 37 ++++ .../gel/ControlFunctionRegistry.java | 2 + .../operations/row/RowReorderOperation.java | 8 +- .../gridworks/templating/DynamicFragment.java | 11 ++ .../gridworks/templating/Fragment.java | 5 + .../metaweb/gridworks/templating/Parser.java | 71 +++++++ .../gridworks/templating/StaticFragment.java | 9 + .../gridworks/templating/Template.java | 173 ++++++++++++++++++ src/main/webapp/project.html | 1 + .../dialogs/templating-exporter-dialog.js | 125 +++++++++++++ src/main/webapp/scripts/project/menu-bar.js | 15 +- .../webapp/scripts/views/data-table-view.js | 4 + src/main/webapp/styles/common.css | 4 + 17 files changed, 589 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/metaweb/gridworks/exporters/TemplatingExporter.java create mode 100644 src/main/java/com/metaweb/gridworks/expr/functions/Jsonize.java create mode 100644 src/main/java/com/metaweb/gridworks/templating/DynamicFragment.java create mode 100644 src/main/java/com/metaweb/gridworks/templating/Fragment.java create mode 100644 src/main/java/com/metaweb/gridworks/templating/Parser.java create mode 100644 src/main/java/com/metaweb/gridworks/templating/StaticFragment.java create mode 100644 src/main/java/com/metaweb/gridworks/templating/Template.java create mode 100644 src/main/webapp/scripts/dialogs/templating-exporter-dialog.js diff --git a/CHANGES.txt b/CHANGES.txt index d909cd7aa..59b883d5c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,8 +23,9 @@ Fixes: Features: - Row/record sorting (Issue 32) -- CSV exporting (Issue 59) -- Mqlwrite exporting +- CSV exporter (Issue 59) +- Mqlwrite exporter +- Templating exporter Changes: - Moved unit tests from JUnit to TestNG diff --git a/src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java index 05b4688d2..21dc91bdd 100644 --- a/src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java @@ -2,6 +2,7 @@ package com.metaweb.gridworks.commands.project; import java.io.IOException; import java.io.PrintWriter; +import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -16,6 +17,7 @@ import com.metaweb.gridworks.commands.Command; import com.metaweb.gridworks.exporters.CsvExporter; import com.metaweb.gridworks.exporters.Exporter; import com.metaweb.gridworks.exporters.HtmlTableExporter; +import com.metaweb.gridworks.exporters.TemplatingExporter; import com.metaweb.gridworks.exporters.ProtographTransposeExporter.TripleLoaderExporter; import com.metaweb.gridworks.exporters.ProtographTransposeExporter.MqlwriteLikeExporter; import com.metaweb.gridworks.exporters.XlsExporter; @@ -29,9 +31,24 @@ public class ExportRowsCommand extends Command { s_formatToExporter.put("html", new HtmlTableExporter()); s_formatToExporter.put("xls", new XlsExporter()); s_formatToExporter.put("csv", new CsvExporter()); + + s_formatToExporter.put("template", new TemplatingExporter()); + s_formatToExporter.put("tripleloader", new TripleLoaderExporter()); s_formatToExporter.put("mqlwrite", new MqlwriteLikeExporter()); } + + @SuppressWarnings("unchecked") + static public Properties getRequestParameters(HttpServletRequest request) { + Properties options = new Properties(); + + Enumeration en = request.getParameterNames(); + while (en.hasMoreElements()) { + String name = en.nextElement(); + options.put(name, request.getParameter(name)); + } + return options; + } public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -41,6 +58,7 @@ public class ExportRowsCommand extends Command { Project project = getProject(request); Engine engine = getEngine(request, project); String format = request.getParameter("format"); + Properties options = getRequestParameters(request); Exporter exporter = s_formatToExporter.get(format.toLowerCase()); if (exporter == null){ @@ -52,9 +70,9 @@ public class ExportRowsCommand extends Command { if (exporter.takeWriter()) { PrintWriter writer = response.getWriter(); - exporter.export(project, new Properties(), engine, writer); + exporter.export(project, options, engine, writer); } else { - exporter.export(project, new Properties(), engine, response.getOutputStream()); + exporter.export(project, options, engine, response.getOutputStream()); } } catch (Exception e) { respondException(response, e); diff --git a/src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java index fe4686f5e..2a2bdbdcd 100644 --- a/src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java @@ -68,7 +68,9 @@ public class GetRowsCommand extends Command { SortingRowVisitor srv = new SortingRowVisitor(visitor); srv.initializeFromJSON(project, sortingJson); - visitor = srv; + if (srv.hasCriteria()) { + visitor = srv; + } } writer.key("mode"); writer.value("row-based"); @@ -85,7 +87,9 @@ public class GetRowsCommand extends Command { SortingRecordVisitor srv = new SortingRecordVisitor(visitor); srv.initializeFromJSON(project, sortingJson); - visitor = srv; + if (srv.hasCriteria()) { + visitor = srv; + } } writer.key("mode"); writer.value("record-based"); diff --git a/src/main/java/com/metaweb/gridworks/exporters/TemplatingExporter.java b/src/main/java/com/metaweb/gridworks/exporters/TemplatingExporter.java new file mode 100644 index 000000000..b50b6443f --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/exporters/TemplatingExporter.java @@ -0,0 +1,101 @@ +package com.metaweb.gridworks.exporters; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRecords; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RecordVisitor; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.browsing.Engine.Mode; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.sorting.SortingRecordVisitor; +import com.metaweb.gridworks.sorting.SortingRowVisitor; +import com.metaweb.gridworks.templating.Parser; +import com.metaweb.gridworks.templating.Template; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class TemplatingExporter implements Exporter { + public String getContentType() { + return "application/x-unknown"; + } + + public boolean takeWriter() { + return true; + } + + public void export(Project project, Properties options, Engine engine, + OutputStream outputStream) throws IOException { + throw new RuntimeException("Not implemented"); + } + + public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException { + String limitString = options.getProperty("limit"); + int limit = limitString != null ? Integer.parseInt(limitString) : -1; + + JSONObject sortingJson = null; + try{ + String json = options.getProperty("sorting"); + sortingJson = (json == null) ? null : + ParsingUtilities.evaluateJsonStringToObject(json); + } catch (JSONException e) { + } + + Template template; + try { + template = Parser.parse(options.getProperty("template")); + } catch (ParsingException e) { + throw new IOException("Missing or bad template", e); + } + + template.setPrefix(options.getProperty("prefix")); + template.setSuffix(options.getProperty("suffix")); + template.setSeparator(options.getProperty("separator")); + + if (engine.getMode() == Mode.RowBased) { + FilteredRows filteredRows = engine.getAllFilteredRows(); + RowVisitor visitor = template.getRowVisitor(writer, limit); + + if (sortingJson != null) { + try { + SortingRowVisitor srv = new SortingRowVisitor(visitor); + srv.initializeFromJSON(project, sortingJson); + + if (srv.hasCriteria()) { + visitor = srv; + } + } catch (JSONException e) { + e.printStackTrace(); + } + } + + filteredRows.accept(project, visitor); + } else { + FilteredRecords filteredRecords = engine.getFilteredRecords(); + RecordVisitor visitor = template.getRecordVisitor(writer, limit); + + if (sortingJson != null) { + try { + SortingRecordVisitor srv = new SortingRecordVisitor(visitor); + srv.initializeFromJSON(project, sortingJson); + + if (srv.hasCriteria()) { + visitor = srv; + } + } catch (JSONException e) { + e.printStackTrace(); + } + } + + filteredRecords.accept(project, visitor); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/Jsonize.java b/src/main/java/com/metaweb/gridworks/expr/functions/Jsonize.java new file mode 100644 index 000000000..c3bdf345a --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/expr/functions/Jsonize.java @@ -0,0 +1,37 @@ +package com.metaweb.gridworks.expr.functions; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.gel.Function; + +public class Jsonize implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length >= 1) { + Object o1 = args[0]; + if (o1 == null) { + return "null"; + } else if (o1 instanceof Number || o1 instanceof Boolean) { + return o1.toString(); + } else { + return JSONObject.quote(o1 instanceof String ? (String) o1 : o1.toString()); + } + } + return null; + } + + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Quotes a value as a JSON literal value"); + writer.key("params"); writer.value("value"); + writer.key("returns"); writer.value("JSON literal value"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java b/src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java index 430d86ebd..b7c3b664c 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java +++ b/src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java @@ -8,6 +8,7 @@ import java.util.Map.Entry; import com.metaweb.gridworks.expr.functions.Cross; import com.metaweb.gridworks.expr.functions.FacetCount; import com.metaweb.gridworks.expr.functions.Get; +import com.metaweb.gridworks.expr.functions.Jsonize; import com.metaweb.gridworks.expr.functions.Length; import com.metaweb.gridworks.expr.functions.Slice; import com.metaweb.gridworks.expr.functions.ToDate; @@ -145,6 +146,7 @@ public class ControlFunctionRegistry { registerFunction("ngramFingerprint", new NGramFingerprint()); registerFunction("phonetic", new Phonetic()); registerFunction("reinterpret", new Reinterpret()); + registerFunction("jsonize", new Jsonize()); registerFunction("indexOf", new IndexOf()); registerFunction("lastIndexOf", new LastIndexOf()); diff --git a/src/main/java/com/metaweb/gridworks/operations/row/RowReorderOperation.java b/src/main/java/com/metaweb/gridworks/operations/row/RowReorderOperation.java index 93349cadd..2244adc1d 100644 --- a/src/main/java/com/metaweb/gridworks/operations/row/RowReorderOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/row/RowReorderOperation.java @@ -65,7 +65,9 @@ public class RowReorderOperation extends AbstractOperation { SortingRowVisitor srv = new SortingRowVisitor(visitor); srv.initializeFromJSON(project, _sorting); - visitor = srv; + if (srv.hasCriteria()) { + visitor = srv; + } } engine.getAllRows().accept(project, visitor); @@ -75,7 +77,9 @@ public class RowReorderOperation extends AbstractOperation { SortingRecordVisitor srv = new SortingRecordVisitor(visitor); srv.initializeFromJSON(project, _sorting); - visitor = srv; + if (srv.hasCriteria()) { + visitor = srv; + } } engine.getAllRecords().accept(project, visitor); diff --git a/src/main/java/com/metaweb/gridworks/templating/DynamicFragment.java b/src/main/java/com/metaweb/gridworks/templating/DynamicFragment.java new file mode 100644 index 000000000..e48eeee4b --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/templating/DynamicFragment.java @@ -0,0 +1,11 @@ +package com.metaweb.gridworks.templating; + +import com.metaweb.gridworks.expr.Evaluable; + +class DynamicFragment extends Fragment { + final public Evaluable eval; + + public DynamicFragment(Evaluable eval) { + this.eval = eval; + } +} diff --git a/src/main/java/com/metaweb/gridworks/templating/Fragment.java b/src/main/java/com/metaweb/gridworks/templating/Fragment.java new file mode 100644 index 000000000..aefb4a206 --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/templating/Fragment.java @@ -0,0 +1,5 @@ +package com.metaweb.gridworks.templating; + +public class Fragment { + +} diff --git a/src/main/java/com/metaweb/gridworks/templating/Parser.java b/src/main/java/com/metaweb/gridworks/templating/Parser.java new file mode 100644 index 000000000..8fe92d84c --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/templating/Parser.java @@ -0,0 +1,71 @@ +package com.metaweb.gridworks.templating; + +import java.util.ArrayList; +import java.util.List; + +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.gel.ast.FieldAccessorExpr; +import com.metaweb.gridworks.gel.ast.VariableExpr; + +public class Parser { + static public Template parse(String s) throws ParsingException { + List fragments = new ArrayList(); + + int start = 0, current = 0; + while (current < s.length() - 1) { + char c = s.charAt(current); + if (c == '\\') { + current += 2; + continue; + } + + char c2 = s.charAt(current + 1); + if (c == '$' && c2 == '{') { + int closeBrace = s.indexOf('}', current + 2); + if (closeBrace > current + 1) { + String columnName = s.substring(current + 2, closeBrace); + + if (current > start) { + fragments.add(new StaticFragment(s.substring(start, current))); + } + start = current = closeBrace + 1; + + fragments.add( + new DynamicFragment( + new FieldAccessorExpr( + new FieldAccessorExpr( + new VariableExpr("cells"), + columnName), + "value"))); + + continue; + } + } else if (c == '{' && c2 == '{') { + int closeBrace = s.indexOf('}', current + 2); + if (closeBrace > current + 1 && closeBrace < s.length() - 1 && s.charAt(closeBrace + 1) == '}') { + String expression = s.substring(current + 2, closeBrace); + + if (current > start) { + fragments.add(new StaticFragment(s.substring(start, current))); + } + start = current = closeBrace + 2; + + fragments.add( + new DynamicFragment( + MetaParser.parse(expression))); + + continue; + } + } + + current++; + } + + if (start < s.length()) { + fragments.add(new StaticFragment(s.substring(start))); + } + + return new Template(fragments); + } +} diff --git a/src/main/java/com/metaweb/gridworks/templating/StaticFragment.java b/src/main/java/com/metaweb/gridworks/templating/StaticFragment.java new file mode 100644 index 000000000..0a9635f5b --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/templating/StaticFragment.java @@ -0,0 +1,9 @@ +package com.metaweb.gridworks.templating; + +class StaticFragment extends Fragment { + final public String text; + + public StaticFragment(String text) { + this.text = text; + } +} diff --git a/src/main/java/com/metaweb/gridworks/templating/Template.java b/src/main/java/com/metaweb/gridworks/templating/Template.java new file mode 100644 index 000000000..988d43a4f --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/templating/Template.java @@ -0,0 +1,173 @@ +package com.metaweb.gridworks.templating; + +import java.io.IOException; +import java.io.Writer; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +import com.metaweb.gridworks.browsing.RecordVisitor; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Record; +import com.metaweb.gridworks.model.Row; + +public class Template { + protected String _prefix; + protected String _suffix; + protected String _separator; + + protected List _fragments; + + public Template(List fragments) { + _fragments = fragments; + } + + public void setPrefix(String prefix) { + _prefix = prefix; + } + + public void setSuffix(String suffix) { + _suffix = suffix; + } + + public void setSeparator(String separator) { + _separator = separator; + } + + public RowVisitor getRowVisitor(Writer writer, int limit) { + return get(writer, limit); + } + + public RecordVisitor getRecordVisitor(Writer writer, int limit) { + return get(writer, limit); + } + + protected RowWritingVisitor get(Writer writer, int limit) { + return new RowWritingVisitor(writer, limit); + } + + protected class RowWritingVisitor implements RowVisitor, RecordVisitor { + final protected int limit; + final protected Writer writer; + protected Properties bindings; + + public int total; + + public RowWritingVisitor(Writer writer, int limit) { + this.limit = limit; + this.writer = writer; + } + + @Override + public void start(Project project) { + bindings = ExpressionUtils.createBindings(project); + + try { + if (_prefix != null) { + writer.write(_prefix); + } + } catch (IOException e) { + // ignore + } + } + + @Override + public void end(Project project) { + try { + if (_suffix != null) { + writer.write(_suffix); + } + } catch (IOException e) { + // ignore + } + } + + public boolean visit(Project project, int rowIndex, Row row) { + if (limit <= 0 || total < limit) { + internalVisit(project, rowIndex, row); + } + total++; + + return limit > 0 && total >= limit; + } + + @Override + public boolean visit(Project project, Record record) { + if (limit <= 0 || total < limit) { + internalVisit(project, record); + } + total++; + + return limit > 0 && total >= limit; + } + + protected void writeValue(Object v) throws IOException { + if (v == null) { + writer.write("null"); + } else if (ExpressionUtils.isError(v)) { + writer.write("null"); + //writer.write("[Error: " + ((EvalError) v).message); + } else if (v instanceof String) { + writer.write((String) v); + } else { + writer.write(v.toString()); + } + } + + public boolean internalVisit(Project project, int rowIndex, Row row) { + try { + if (total > 0 && _separator != null) { + writer.write(_separator); + } + + ExpressionUtils.bind(bindings, row, rowIndex, null, null); + for (Fragment f : _fragments) { + if (f instanceof StaticFragment) { + writer.write(((StaticFragment) f).text); + } else { + DynamicFragment df = (DynamicFragment) f; + Object value = df.eval.evaluate(bindings); + + if (value != null && ExpressionUtils.isArrayOrCollection(value)) { + if (ExpressionUtils.isArray(value)) { + Object[] a = (Object[]) value; + for (Object v : a) { + writeValue(v); + } + } else { + Collection a = ExpressionUtils.toObjectCollection(value); + for (Object v : a) { + writeValue(v); + } + } + continue; + } + + writeValue(value); + } + } + } catch (IOException e) { + // ignore + } + return false; + } + + protected boolean internalVisit(Project project, Record record) { + bindings.put("recordIndex", record.recordIndex); + + for (int r = record.fromRowIndex; r < record.toRowIndex; r++) { + Row row = project.rows.get(r); + + bindings.put("rowIndex", r); + + internalVisit(project, r, row); + + bindings.remove("recordIndex"); + } + return false; + } + } + +} diff --git a/src/main/webapp/project.html b/src/main/webapp/project.html index 8e316bd1c..a9ff18eca 100644 --- a/src/main/webapp/project.html +++ b/src/main/webapp/project.html @@ -80,6 +80,7 @@ + diff --git a/src/main/webapp/scripts/dialogs/templating-exporter-dialog.js b/src/main/webapp/scripts/dialogs/templating-exporter-dialog.js new file mode 100644 index 000000000..6e26a9401 --- /dev/null +++ b/src/main/webapp/scripts/dialogs/templating-exporter-dialog.js @@ -0,0 +1,125 @@ +function TemplatingExporterDialog() { + this._timerID = null; + this._createDialog(); + this._updatePreview(); +} + +TemplatingExporterDialog.prototype._createDialog = function() { + var self = this; + var frame = DialogSystem.createDialog(); + frame.width("900px"); + + var header = $('
').addClass("dialog-header").text('Templating Export').appendTo(frame); + var body = $('
').addClass("dialog-body").appendTo(frame); + var footer = $('
').addClass("dialog-footer").appendTo(frame); + + body.html( + '
' + + '' + + '' + + '' + + '' + + '
' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '
Prefix
Row Template
Row Separator
Suffix
' + + '
' + + '
' + ); + + this._elmts = DOM.bind(body); + this._elmts.controls.find("textarea").keyup(function() { self._scheduleUpdate(); }); + + this._elmts.prefixTextarea[0].value = '{\n "rows" : [\n'; + this._elmts.suffixTextarea[0].value = '\n ]\n}'; + this._elmts.separatorTextarea[0].value = ',\n'; + this._elmts.templateTextarea[0].value = ' {' + + $.map(theProject.columnModel.columns, function(column, i) { + return '\n "' + column.name + '" : {{jsonize(cells["' + column.name + '"].value)}}'; + }).join(',') + '\n }'; + + $('').text("Export").click(function() { self._export(); self._dismiss(); }).appendTo(footer); + $('').text("Close").click(function() { self._dismiss(); }).appendTo(footer); + + this._level = DialogSystem.showDialog(frame); +}; + +TemplatingExporterDialog.prototype._scheduleUpdate = function() { + var self = this; + + if (this._timerID) { + window.clearTimeout(this._timerID); + } + + this._elmts.previewTextarea[0].value = "Idling..."; + this._timerID = window.setTimeout(function() { + self._timerID = null; + self._elmts.previewTextarea[0].value = "Updating..."; + self._updatePreview(); + }, 1000); +}; + +TemplatingExporterDialog.prototype._dismiss = function() { + DialogSystem.dismissUntil(this._level - 1); +}; + +TemplatingExporterDialog.prototype._updatePreview = function() { + var self = this; + $.post( + "/command/export-rows/preview.txt", + { + "project" : theProject.id, + "format" : "template", + "engine" : JSON.stringify(ui.browsingEngine.getJSON()), + "sorting" : JSON.stringify(ui.dataTableView.getSorting()), + "prefix" : this._elmts.prefixTextarea[0].value, + "suffix" : this._elmts.suffixTextarea[0].value, + "separator" : this._elmts.separatorTextarea[0].value, + "template" : this._elmts.templateTextarea[0].value, + "limit" : "20" + }, + function (data) { + self._elmts.previewTextarea[0].value = data; + }, + "text" + ); +}; + +TemplatingExporterDialog.prototype._export = function() { + var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "post") + .attr("action", "/command/export-rows/" + name + ".txt") + .attr("target", "gridworks-export"); + + var appendField = function(name, value) { + $('