s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial implementation of a templating exporter.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@860 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-05-26 05:42:31 +00:00
parent 76cb65bfb8
commit 017a825600
17 changed files with 589 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGES.txt
View File

@ -23,8 +23,9 @@ Fixes:
Features:
- Row/record sorting (Issue 32)
- CSV exporting (Issue 59)
- Mqlwrite exporting
- CSV exporter (Issue 59)
- Mqlwrite exporter
- Templating exporter
Changes:
- Moved unit tests from JUnit to TestNG

22
src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java
View File

@ -2,6 +2,7 @@ package com.metaweb.gridworks.commands.project;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@ -16,6 +17,7 @@ import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.exporters.CsvExporter;
import com.metaweb.gridworks.exporters.Exporter;
import com.metaweb.gridworks.exporters.HtmlTableExporter;
import com.metaweb.gridworks.exporters.TemplatingExporter;
import com.metaweb.gridworks.exporters.ProtographTransposeExporter.TripleLoaderExporter;
import com.metaweb.gridworks.exporters.ProtographTransposeExporter.MqlwriteLikeExporter;
import com.metaweb.gridworks.exporters.XlsExporter;
@ -29,9 +31,24 @@ public class ExportRowsCommand extends Command {
s_formatToExporter.put("html", new HtmlTableExporter());
s_formatToExporter.put("xls", new XlsExporter());
s_formatToExporter.put("csv", new CsvExporter());
s_formatToExporter.put("template", new TemplatingExporter());
s_formatToExporter.put("tripleloader", new TripleLoaderExporter());
s_formatToExporter.put("mqlwrite", new MqlwriteLikeExporter());
}
@SuppressWarnings("unchecked")
static public Properties getRequestParameters(HttpServletRequest request) {
Properties options = new Properties();
Enumeration<String> en = request.getParameterNames();
while (en.hasMoreElements()) {
String name = en.nextElement();
options.put(name, request.getParameter(name));
}
return options;
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -41,6 +58,7 @@ public class ExportRowsCommand extends Command {
Project project = getProject(request);
Engine engine = getEngine(request, project);
String format = request.getParameter("format");
Properties options = getRequestParameters(request);
Exporter exporter = s_formatToExporter.get(format.toLowerCase());
if (exporter == null){
@ -52,9 +70,9 @@ public class ExportRowsCommand extends Command {
if (exporter.takeWriter()) {
PrintWriter writer = response.getWriter();
exporter.export(project, new Properties(), engine, writer);
exporter.export(project, options, engine, writer);
} else {
exporter.export(project, new Properties(), engine, response.getOutputStream());
exporter.export(project, options, engine, response.getOutputStream());
}
} catch (Exception e) {
respondException(response, e);

8
src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java
View File

@ -68,7 +68,9 @@ public class GetRowsCommand extends Command {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
visitor = srv;
if (srv.hasCriteria()) {
visitor = srv;
}
}
writer.key("mode"); writer.value("row-based");
@ -85,7 +87,9 @@ public class GetRowsCommand extends Command {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
visitor = srv;
if (srv.hasCriteria()) {
visitor = srv;
}
}
writer.key("mode"); writer.value("record-based");

101
src/main/java/com/metaweb/gridworks/exporters/TemplatingExporter.java Normal file
View File

@ -0,0 +1,101 @@
package com.metaweb.gridworks.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.browsing.Engine.Mode;
import com.metaweb.gridworks.expr.ParsingException;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.sorting.SortingRecordVisitor;
import com.metaweb.gridworks.sorting.SortingRowVisitor;
import com.metaweb.gridworks.templating.Parser;
import com.metaweb.gridworks.templating.Template;
import com.metaweb.gridworks.util.ParsingUtilities;
public class TemplatingExporter implements Exporter {
public String getContentType() {
return "application/x-unknown";
}
public boolean takeWriter() {
return true;
}
public void export(Project project, Properties options, Engine engine,
OutputStream outputStream) throws IOException {
throw new RuntimeException("Not implemented");
}
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
String limitString = options.getProperty("limit");
int limit = limitString != null ? Integer.parseInt(limitString) : -1;
JSONObject sortingJson = null;
try{
String json = options.getProperty("sorting");
sortingJson = (json == null) ? null :
ParsingUtilities.evaluateJsonStringToObject(json);
} catch (JSONException e) {
}
Template template;
try {
template = Parser.parse(options.getProperty("template"));
} catch (ParsingException e) {
throw new IOException("Missing or bad template", e);
}
template.setPrefix(options.getProperty("prefix"));
template.setSuffix(options.getProperty("suffix"));
template.setSeparator(options.getProperty("separator"));
if (engine.getMode() == Mode.RowBased) {
FilteredRows filteredRows = engine.getAllFilteredRows();
RowVisitor visitor = template.getRowVisitor(writer, limit);
if (sortingJson != null) {
try {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
if (srv.hasCriteria()) {
visitor = srv;
}
} catch (JSONException e) {
e.printStackTrace();
}
}
filteredRows.accept(project, visitor);
} else {
FilteredRecords filteredRecords = engine.getFilteredRecords();
RecordVisitor visitor = template.getRecordVisitor(writer, limit);
if (sortingJson != null) {
try {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
if (srv.hasCriteria()) {
visitor = srv;
}
} catch (JSONException e) {
e.printStackTrace();
}
}
filteredRecords.accept(project, visitor);
}
}
}

37
src/main/java/com/metaweb/gridworks/expr/functions/Jsonize.java Normal file
View File

@ -0,0 +1,37 @@
package com.metaweb.gridworks.expr.functions;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.gel.Function;
public class Jsonize implements Function {
public Object call(Properties bindings, Object[] args) {
if (args.length >= 1) {
Object o1 = args[0];
if (o1 == null) {
return "null";
} else if (o1 instanceof Number || o1 instanceof Boolean) {
return o1.toString();
} else {
return JSONObject.quote(o1 instanceof String ? (String) o1 : o1.toString());
}
}
return null;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value("Quotes a value as a JSON literal value");
writer.key("params"); writer.value("value");
writer.key("returns"); writer.value("JSON literal value");
writer.endObject();
}
}

2
src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java
View File

@ -8,6 +8,7 @@ import java.util.Map.Entry;
import com.metaweb.gridworks.expr.functions.Cross;
import com.metaweb.gridworks.expr.functions.FacetCount;
import com.metaweb.gridworks.expr.functions.Get;
import com.metaweb.gridworks.expr.functions.Jsonize;
import com.metaweb.gridworks.expr.functions.Length;
import com.metaweb.gridworks.expr.functions.Slice;
import com.metaweb.gridworks.expr.functions.ToDate;
@ -145,6 +146,7 @@ public class ControlFunctionRegistry {
registerFunction("ngramFingerprint", new NGramFingerprint());
registerFunction("phonetic", new Phonetic());
registerFunction("reinterpret", new Reinterpret());
registerFunction("jsonize", new Jsonize());
registerFunction("indexOf", new IndexOf());
registerFunction("lastIndexOf", new LastIndexOf());

8
src/main/java/com/metaweb/gridworks/operations/row/RowReorderOperation.java
View File

@ -65,7 +65,9 @@ public class RowReorderOperation extends AbstractOperation {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
visitor = srv;
if (srv.hasCriteria()) {
visitor = srv;
}
}
engine.getAllRows().accept(project, visitor);
@ -75,7 +77,9 @@ public class RowReorderOperation extends AbstractOperation {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
visitor = srv;
if (srv.hasCriteria()) {
visitor = srv;
}
}
engine.getAllRecords().accept(project, visitor);

11
src/main/java/com/metaweb/gridworks/templating/DynamicFragment.java Normal file
View File

@ -0,0 +1,11 @@
package com.metaweb.gridworks.templating;
import com.metaweb.gridworks.expr.Evaluable;
class DynamicFragment extends Fragment {
final public Evaluable eval;
public DynamicFragment(Evaluable eval) {
this.eval = eval;
}
}

5
src/main/java/com/metaweb/gridworks/templating/Fragment.java Normal file
View File

@ -0,0 +1,5 @@
package com.metaweb.gridworks.templating;
public class Fragment {
}

71
src/main/java/com/metaweb/gridworks/templating/Parser.java Normal file
View File

@ -0,0 +1,71 @@
package com.metaweb.gridworks.templating;
import java.util.ArrayList;
import java.util.List;
import com.metaweb.gridworks.expr.MetaParser;
import com.metaweb.gridworks.expr.ParsingException;
import com.metaweb.gridworks.gel.ast.FieldAccessorExpr;
import com.metaweb.gridworks.gel.ast.VariableExpr;
public class Parser {
static public Template parse(String s) throws ParsingException {
List<Fragment> fragments = new ArrayList<Fragment>();
int start = 0, current = 0;
while (current < s.length() - 1) {
char c = s.charAt(current);
if (c == '\\') {
current += 2;
continue;
}
char c2 = s.charAt(current + 1);
if (c == '$' && c2 == '{') {
int closeBrace = s.indexOf('}', current + 2);
if (closeBrace > current + 1) {
String columnName = s.substring(current + 2, closeBrace);
if (current > start) {
fragments.add(new StaticFragment(s.substring(start, current)));
}
start = current = closeBrace + 1;
fragments.add(
new DynamicFragment(
new FieldAccessorExpr(
new FieldAccessorExpr(
new VariableExpr("cells"),
columnName),
"value")));
continue;
}
} else if (c == '{' && c2 == '{') {
int closeBrace = s.indexOf('}', current + 2);
if (closeBrace > current + 1 && closeBrace < s.length() - 1 && s.charAt(closeBrace + 1) == '}') {
String expression = s.substring(current + 2, closeBrace);
if (current > start) {
fragments.add(new StaticFragment(s.substring(start, current)));
}
start = current = closeBrace + 2;
fragments.add(
new DynamicFragment(
MetaParser.parse(expression)));
continue;
}
}
current++;
}
if (start < s.length()) {
fragments.add(new StaticFragment(s.substring(start)));
}
return new Template(fragments);
}
}

9
src/main/java/com/metaweb/gridworks/templating/StaticFragment.java Normal file
View File

@ -0,0 +1,9 @@
package com.metaweb.gridworks.templating;
class StaticFragment extends Fragment {
final public String text;
public StaticFragment(String text) {
this.text = text;
}
}

173
src/main/java/com/metaweb/gridworks/templating/Template.java Normal file
View File

@ -0,0 +1,173 @@
package com.metaweb.gridworks.templating;
import java.io.IOException;
import java.io.Writer;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
public class Template {
protected String _prefix;
protected String _suffix;
protected String _separator;
protected List<Fragment> _fragments;
public Template(List<Fragment> fragments) {
_fragments = fragments;
}
public void setPrefix(String prefix) {
_prefix = prefix;
}
public void setSuffix(String suffix) {
_suffix = suffix;
}
public void setSeparator(String separator) {
_separator = separator;
}
public RowVisitor getRowVisitor(Writer writer, int limit) {
return get(writer, limit);
}
public RecordVisitor getRecordVisitor(Writer writer, int limit) {
return get(writer, limit);
}
protected RowWritingVisitor get(Writer writer, int limit) {
return new RowWritingVisitor(writer, limit);
}
protected class RowWritingVisitor implements RowVisitor, RecordVisitor {
final protected int limit;
final protected Writer writer;
protected Properties bindings;
public int total;
public RowWritingVisitor(Writer writer, int limit) {
this.limit = limit;
this.writer = writer;
}
@Override
public void start(Project project) {
bindings = ExpressionUtils.createBindings(project);
try {
if (_prefix != null) {
writer.write(_prefix);
}
} catch (IOException e) {
// ignore
}
}
@Override
public void end(Project project) {
try {
if (_suffix != null) {
writer.write(_suffix);
}
} catch (IOException e) {
// ignore
}
}
public boolean visit(Project project, int rowIndex, Row row) {
if (limit <= 0 || total < limit) {
internalVisit(project, rowIndex, row);
}
total++;
return limit > 0 && total >= limit;
}
@Override
public boolean visit(Project project, Record record) {
if (limit <= 0 || total < limit) {
internalVisit(project, record);
}
total++;
return limit > 0 && total >= limit;
}
protected void writeValue(Object v) throws IOException {
if (v == null) {
writer.write("null");
} else if (ExpressionUtils.isError(v)) {
writer.write("null");
//writer.write("[Error: " + ((EvalError) v).message);
} else if (v instanceof String) {
writer.write((String) v);
} else {
writer.write(v.toString());
}
}
public boolean internalVisit(Project project, int rowIndex, Row row) {
try {
if (total > 0 && _separator != null) {
writer.write(_separator);
}
ExpressionUtils.bind(bindings, row, rowIndex, null, null);
for (Fragment f : _fragments) {
if (f instanceof StaticFragment) {
writer.write(((StaticFragment) f).text);
} else {
DynamicFragment df = (DynamicFragment) f;
Object value = df.eval.evaluate(bindings);
if (value != null && ExpressionUtils.isArrayOrCollection(value)) {
if (ExpressionUtils.isArray(value)) {
Object[] a = (Object[]) value;
for (Object v : a) {
writeValue(v);
}
} else {
Collection<Object> a = ExpressionUtils.toObjectCollection(value);
for (Object v : a) {
writeValue(v);
}
}
continue;
}
writeValue(value);
}
}
} catch (IOException e) {
// ignore
}
return false;
}
protected boolean internalVisit(Project project, Record record) {
bindings.put("recordIndex", record.recordIndex);
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
Row row = project.rows.get(r);
bindings.put("rowIndex", r);
internalVisit(project, r, row);
bindings.remove("recordIndex");
}
return false;
}
}
}

1
src/main/webapp/project.html
View File

@ -80,6 +80,7 @@
<script type="text/javascript" src="scripts/dialogs/clustering-dialog.js"></script>
<script type="text/javascript" src="scripts/dialogs/scatterplot-dialog.js"></script>
<script type="text/javascript" src="scripts/dialogs/extend-data-preview-dialog.js"></script>
<script type="text/javascript" src="scripts/dialogs/templating-exporter-dialog.js"></script>
<script type="text/javascript" src="scripts/protograph/schema-alignment.js"></script>
<script type="text/javascript" src="scripts/protograph/schema-alignment-ui-node.js"></script>

125
src/main/webapp/scripts/dialogs/templating-exporter-dialog.js Normal file
View File

@ -0,0 +1,125 @@
function TemplatingExporterDialog() {
this._timerID = null;
this._createDialog();
this._updatePreview();
}
TemplatingExporterDialog.prototype._createDialog = function() {
var self = this;
var frame = DialogSystem.createDialog();
frame.width("900px");
var header = $('<div></div>').addClass("dialog-header").text('Templating Export').appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div>').addClass("dialog-footer").appendTo(frame);
body.html(
'<div class="grid-layout layout-normal layout-full"><table>' +
'<tr>' +
'<td style="vertical-align: top"><div class="grid-layout layout-tighter layout-full" bind="controls"><table>' +
'<tr><td>Prefix</td></tr>' +
'<tr><td><div class="input-container"><textarea bind="prefixTextarea" class="code" wrap="off" style="height:5em;"></textarea></div></td></tr>' +
'<tr><td>Row Template</td></tr>' +
'<tr><td><div class="input-container"><textarea bind="templateTextarea" class="code" wrap="off" style="height:20em;"></textarea></div></td></tr>' +
'<tr><td>Row Separator</td></tr>' +
'<tr><td><div class="input-container"><textarea bind="separatorTextarea" class="code" wrap="off" style="height:3em;"></textarea></div></td></tr>' +
'<tr><td>Suffix</td></tr>' +
'<tr><td><div class="input-container"><textarea bind="suffixTextarea" class="code" wrap="off" style="height:5em;"></textarea></div></td></tr>' +
'</table></div></td>' +
'<td width="50%" style="vertical-align: top">' +
'<div class="input-container"><textarea bind="previewTextarea" class="code" wrap="off" style="height: 40em;"></textarea></div>' +
'</td>' +
'</tr>' +
'</table></div>'
);
this._elmts = DOM.bind(body);
this._elmts.controls.find("textarea").keyup(function() { self._scheduleUpdate(); });
this._elmts.prefixTextarea[0].value = '{\n "rows" : [\n';
this._elmts.suffixTextarea[0].value = '\n ]\n}';
this._elmts.separatorTextarea[0].value = ',\n';
this._elmts.templateTextarea[0].value = ' {' +
$.map(theProject.columnModel.columns, function(column, i) {
return '\n "' + column.name + '" : {{jsonize(cells["' + column.name + '"].value)}}';
}).join(',') + '\n }';
$('<button></button>').text("Export").click(function() { self._export(); self._dismiss(); }).appendTo(footer);
$('<button></button>').text("Close").click(function() { self._dismiss(); }).appendTo(footer);
this._level = DialogSystem.showDialog(frame);
};
TemplatingExporterDialog.prototype._scheduleUpdate = function() {
var self = this;
if (this._timerID) {
window.clearTimeout(this._timerID);
}
this._elmts.previewTextarea[0].value = "Idling...";
this._timerID = window.setTimeout(function() {
self._timerID = null;
self._elmts.previewTextarea[0].value = "Updating...";
self._updatePreview();
}, 1000);
};
TemplatingExporterDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
TemplatingExporterDialog.prototype._updatePreview = function() {
var self = this;
$.post(
"/command/export-rows/preview.txt",
{
"project" : theProject.id,
"format" : "template",
"engine" : JSON.stringify(ui.browsingEngine.getJSON()),
"sorting" : JSON.stringify(ui.dataTableView.getSorting()),
"prefix" : this._elmts.prefixTextarea[0].value,
"suffix" : this._elmts.suffixTextarea[0].value,
"separator" : this._elmts.separatorTextarea[0].value,
"template" : this._elmts.templateTextarea[0].value,
"limit" : "20"
},
function (data) {
self._elmts.previewTextarea[0].value = data;
},
"text"
);
};
TemplatingExporterDialog.prototype._export = function() {
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
var form = document.createElement("form");
$(form)
.css("display", "none")
.attr("method", "post")
.attr("action", "/command/export-rows/" + name + ".txt")
.attr("target", "gridworks-export");
var appendField = function(name, value) {
$('<textarea />')
.attr("name", name)
.attr("value", value)
.appendTo(form);
}
appendField("engine", JSON.stringify(ui.browsingEngine.getJSON()));
appendField("project", theProject.id);
appendField("format", "template");
appendField("sorting", JSON.stringify(ui.dataTableView.getSorting()));
appendField("prefix", this._elmts.prefixTextarea[0].value);
appendField("suffix", this._elmts.suffixTextarea[0].value);
appendField("separator", this._elmts.separatorTextarea[0].value);
appendField("template", this._elmts.templateTextarea[0].value);
document.body.appendChild(form);
window.open("about:blank", "gridworks-export");
form.submit();
document.body.removeChild(form);
};

15
src/main/webapp/scripts/project/menu-bar.js
View File

@ -40,10 +40,10 @@ MenuBar.prototype._initializeUI = function() {
"label": "Tab-Separated Value",
"click": function() { self._doExportRows("tsv", "tsv"); }
},
{
"label": "Comma-Separated Value",
"click": function() { self._doExportRows("csv", "csv"); }
},
{
"label": "Comma-Separated Value",
"click": function() { self._doExportRows("csv", "csv"); }
},
{
"label": "HTML Table",
"click": function() { self._doExportRows("html", "html"); }
@ -60,6 +60,13 @@ MenuBar.prototype._initializeUI = function() {
{
"label": "MQLWrite",
"click": function() { self._doExportTripleloader("mqlwrite"); }
},
{},
{
"label": "Templating...",
"click": function() {
new TemplatingExporterDialog();
}
}
]
},

4
src/main/webapp/scripts/views/data-table-view.js
View File

@ -10,6 +10,10 @@ function DataTableView(div) {
this._showRows(0);
}
DataTableView.prototype.getSorting = function() {
return this._sorting;
};
DataTableView.prototype.resize = function() {
var topHeight = this._div.find(".viewPanel-header").outerHeight(true);

4
src/main/webapp/styles/common.css
View File

@ -94,6 +94,10 @@ div.input-container > input, div.input-container > textarea {
vertical-align: top;
}
input.code, textarea.code {
font-family: monospace;
}
a.action,
a.quiet-link {
text-decoration: none;