More scaffolding. We can now upload a tsv/csv file and create and persist a project.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@3 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-01-25 22:51:25 +00:00
parent 22040a8348
commit e1fa9279eb
25 changed files with 4750 additions and 30 deletions

View File

@ -63,11 +63,6 @@
<artifactId>commons-lang</artifactId> <artifactId>commons-lang</artifactId>
<version>2.1</version> <version>2.1</version>
</dependency> </dependency>
<dependency>
<groupId>rhino</groupId>
<artifactId>js</artifactId>
<version>1.7R2</version>
</dependency>
</dependencies> </dependencies>
</project> </project>

View File

@ -1,6 +1,9 @@
package com.metaweb.gridlock; package com.metaweb.gridlock;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
@ -11,22 +14,54 @@ import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONTokener; import org.json.JSONTokener;
import com.metaweb.gridlock.commands.Command;
import com.metaweb.gridlock.commands.CreateProjectFromUploadCommand;
import com.metaweb.gridlock.commands.GetColumnModelCommand;
import com.metaweb.gridlock.commands.GetRowsCommand;
public class GridlockServlet extends HttpServlet { public class GridlockServlet extends HttpServlet {
private static final long serialVersionUID = 2386057901503517403L; private static final long serialVersionUID = 2386057901503517403L;
static private boolean s_verbose = false; static protected Map<String, Command> _commands = new HashMap<String, Command>();
static {
_commands.put("create-project-from-upload", new CreateProjectFromUploadCommand());
_commands.put("get-column-model", new GetColumnModelCommand());
_commands.put("get-rows", new GetRowsCommand());
}
@Override @Override
public void init() throws ServletException { public void init() throws ServletException {
super.init(); super.init();
}
s_verbose = "true".equalsIgnoreCase(System.getProperty("verbose")); @Override
public void destroy() {
ProjectManager.singleton.saveAllProjects();
ProjectManager.singleton.save();
ProjectManager.singleton = null;
super.destroy();
} }
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize(new File("./data"));
String commandName = request.getPathInfo().substring(1);
Command command = _commands.get(commandName);
if (command != null) {
command.doPost(request, response);
}
} }
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize(new File("./data"));
String commandName = request.getPathInfo().substring(1);
Command command = _commands.get(commandName);
if (command != null) {
command.doGet(request, response);
}
} }
static public JSONObject evaluateJsonStringToObject(String s) throws JSONException { static public JSONObject evaluateJsonStringToObject(String s) throws JSONException {
@ -38,10 +73,4 @@ public class GridlockServlet extends HttpServlet {
protected String encodeString(String s) { protected String encodeString(String s) {
return s.replace("\"", "\\\""); return s.replace("\"", "\\\"");
} }
protected void logException(Exception e) {
if (s_verbose) {
e.printStackTrace();
}
}
} }

View File

@ -0,0 +1,191 @@
package com.metaweb.gridlock;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import com.metaweb.gridlock.model.Project;
public class ProjectManager implements Serializable {
private static final long serialVersionUID = -2967415873336723962L;
protected File _dir;
protected Map<Long, ProjectMetadata> _projectsMetadata = new HashMap<Long, ProjectMetadata>();
transient protected Map<Long, Project> _projects;
static public ProjectManager singleton;
static public void initialize(File dir) {
if (singleton == null) {
File file = new File(dir, "projects");
if (file.exists()) {
singleton = load(file);
}
if (singleton == null) {
singleton = new ProjectManager(dir);
}
}
}
static protected ProjectManager load(File file) {
ProjectManager pm = null;
FileInputStream fis = null;
ObjectInputStream in = null;
try {
fis = new FileInputStream(file);
in = new ObjectInputStream(fis);
pm = (ProjectManager) in.readObject();
} catch(IOException e) {
e.printStackTrace();
} catch(ClassNotFoundException e) {
e.printStackTrace();
} finally {
if (fis != null) {
try {
fis.close();
} catch (Exception e) {
}
}
if (in != null) {
try {
in.close();
} catch (Exception e) {
}
}
}
return pm;
}
private ProjectManager(File dir) {
_dir = dir;
_dir.mkdirs();
internalInitialize();
}
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();
internalInitialize();
}
private void internalInitialize() {
_projects = new HashMap<Long, Project>();
}
public Project createProject(ProjectMetadata projectMetadata) {
Project project = new Project();
_projects.put(project.id, project);
_projectsMetadata.put(project.id, projectMetadata);
return project;
}
public Project getProject(long id) {
if (_projects.containsKey(id)) {
return _projects.get(id);
} else {
File file = new File(_dir, id + ".project");
Project project = null;
FileInputStream fis = null;
ObjectInputStream in = null;
try {
fis = new FileInputStream(file);
in = new ObjectInputStream(fis);
project = (Project) in.readObject();
} catch(IOException e) {
e.printStackTrace();
} catch(ClassNotFoundException e) {
e.printStackTrace();
} finally {
if (fis != null) {
try {
fis.close();
} catch (Exception e) {
}
}
if (in != null) {
try {
in.close();
} catch (Exception e) {
}
}
}
_projects.put(id, project);
return project;
}
}
public void save() {
File file = new File(_dir, "projects");
FileOutputStream fos = null;
ObjectOutputStream out = null;
try {
fos = new FileOutputStream(file);
out = new ObjectOutputStream(fos);
out.writeObject(this);
out.flush();
} catch(IOException e) {
e.printStackTrace();
} finally {
if (fos != null) {
try {
fos.close();
} catch (Exception e) {
}
}
if (out != null) {
try {
out.close();
} catch (Exception e) {
}
}
}
}
public void saveAllProjects() {
for (Project project : _projects.values()) {
saveProject(project);
}
}
protected void saveProject(Project project) {
File file = new File(_dir, project.id + ".project");
FileOutputStream fos = null;
ObjectOutputStream out = null;
try {
fos = new FileOutputStream(file);
out = new ObjectOutputStream(fos);
out.writeObject(project);
out.flush();
} catch(IOException e) {
e.printStackTrace();
} finally {
if (fos != null) {
try {
fos.close();
} catch (Exception e) {
}
}
if (out != null) {
try {
out.close();
} catch (Exception e) {
}
}
}
}
}

View File

@ -0,0 +1,10 @@
package com.metaweb.gridlock;
import java.io.Serializable;
public class ProjectMetadata implements Serializable {
private static final long serialVersionUID = 7959027046468240844L;
public String name;
public String password;
}

View File

@ -0,0 +1,135 @@
package com.metaweb.gridlock.commands;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.NotImplementedException;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import com.metaweb.gridlock.ProjectManager;
import com.metaweb.gridlock.model.Project;
import com.oreilly.servlet.multipart.FilePart;
import com.oreilly.servlet.multipart.MultipartParser;
import com.oreilly.servlet.multipart.ParamPart;
import com.oreilly.servlet.multipart.Part;
public abstract class Command {
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
throw new NotImplementedException();
};
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
throw new NotImplementedException();
};
protected Project getProject(HttpServletRequest request) throws ServletException {
try {
Project p = ProjectManager.singleton.getProject(Long.parseLong(request.getParameter("project")));
if (p != null) {
return p;
}
} catch (Exception e) {
// ignore
}
throw new ServletException("Missing or bad project URL parameter");
}
protected int getIntegerParameter(HttpServletRequest request, String name, int def) throws ServletException {
try {
return Integer.parseInt(request.getParameter(name));
} catch (Exception e) {
// ignore
}
return def;
}
protected void respond(HttpServletResponse response, String content) throws IOException {
response.setStatus(HttpServletResponse.SC_OK);
OutputStream os = response.getOutputStream();
OutputStreamWriter osw = new OutputStreamWriter(os);
try {
osw.write(content);
} finally {
osw.flush();
osw.close();
}
}
protected void respondJSON(HttpServletResponse response, JSONObject o) throws IOException {
respond(response, o.toString());
}
protected void respondException(HttpServletResponse response, Exception e) throws IOException {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
e.printStackTrace(response.getWriter());
}
protected void redirect(HttpServletResponse response, String url) throws IOException {
response.setStatus(HttpServletResponse.SC_OK);
String content = "<html><head><meta http-equiv=\"refresh\" content=\"1;url=" + url + "\"></head><body></body></html>";
response.getWriter().print(content);
}
protected String readFileUpload(HttpServletRequest request, Properties properties) throws IOException {
StringBuffer sb = new StringBuffer();
try {
MultipartParser parser = new MultipartParser(request, 20 * 1024 * 1024);
Part part = null;
while ((part = parser.readNextPart()) != null) {
if (part.isFile()) {
Reader reader = new InputStreamReader(((FilePart) part).getInputStream());
LineNumberReader lnr = new LineNumberReader(reader);
try {
String line = null;
while ((line = lnr.readLine()) != null) {
sb.append(line);
sb.append('\n');
}
} finally {
lnr.close();
}
} else if (part.isParam()) {
ParamPart paramPart = (ParamPart) part;
properties.put(part.getName(), paramPart.getStringValue());
}
}
} catch (Exception e) {
e.printStackTrace();
}
return sb.toString();
}
protected JSONObject getJsonParameter(HttpServletRequest request, String name) {
String value = request.getParameter(name);
if (value != null) {
try {
JSONObject o = jsonStringToObject(value);
return o;
} catch (JSONException e) {
}
}
return null;
}
protected JSONObject jsonStringToObject(String s) throws JSONException {
JSONTokener t = new JSONTokener(s);
JSONObject o = (JSONObject) t.nextValue();
return o;
}
}

View File

@ -0,0 +1,110 @@
package com.metaweb.gridlock.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridlock.ProjectManager;
import com.metaweb.gridlock.ProjectMetadata;
import com.metaweb.gridlock.model.Cell;
import com.metaweb.gridlock.model.Column;
import com.metaweb.gridlock.model.Project;
import com.metaweb.gridlock.model.Row;
public class CreateProjectFromUploadCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Properties properties = new Properties();
String content = readFileUpload(request, properties);
ProjectMetadata pm = new ProjectMetadata();
pm.name = properties.getProperty("name");
pm.password = properties.getProperty("password");
Project project = ProjectManager.singleton.createProject(pm);
int start = 0;
String sep = null;
String line = null;
boolean first = true;
int cellCount = 1;
while (start < content.length()) {
int newline = content.indexOf('\n', start);
if (newline < 0) {
line = content.substring(start);
start = content.length();
} else {
line = content.substring(start, newline);
start = newline + 1;
}
if (sep == null) {
int tab = line.indexOf('\t');
if (tab >= 0) {
sep = "\t";
} else {
sep = "\\,";
}
}
String[] cells = line.split(sep);
if (first) {
first = false;
for (int c = 0; c < cells.length; c++) {
String cell = cells[c];
if (cell.startsWith("\"") && cell.endsWith("\"")) {
cell = cell.substring(1, cell.length() - 1);
}
Column column = new Column();
column.cellIndex = c;
column.headerLabel = cell;
project.columnModel.columns.add(column);
}
cellCount = cells.length;
} else {
Row row = new Row(cellCount);
for (int c = 0; c < cells.length; c++) {
String text = cells[c];
Cell cell = new Cell();
cell.value = parseCellValue(text);
row.cells.add(cell);
}
project.rows.add(row);
}
}
redirect(response, "/project.html?project=" + project.id);
}
static public Object parseCellValue(String text) {
if (text.startsWith("\"") && text.endsWith("\"")) {
return text.substring(1, text.length() - 1);
}
try {
return Long.parseLong(text);
} catch (NumberFormatException e) {
}
try {
return Double.parseDouble(text);
} catch (NumberFormatException e) {
}
return text;
}
}

View File

@ -0,0 +1,26 @@
package com.metaweb.gridlock.commands;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import com.metaweb.gridlock.model.Project;
public class GetColumnModelCommand extends Command {
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Project project = getProject(request);
try {
respondJSON(response, project.columnModel.getJSON());
} catch (JSONException e) {
respondException(response, e);
}
}
}

View File

@ -0,0 +1,49 @@
package com.metaweb.gridlock.commands;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import com.metaweb.gridlock.model.Column;
import com.metaweb.gridlock.model.Project;
import com.metaweb.gridlock.model.Row;
public class GetRowsCommand extends Command {
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Project project = getProject(request);
int start = Math.min(project.rows.size(), Math.max(0, getIntegerParameter(request, "start", 0)));
int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20)));
try {
JSONObject o = new JSONObject();
List<JSONObject> a = new ArrayList<JSONObject>(limit);
try {
for (int r = start; r < start + limit && r < project.rows.size(); r++) {
Row row = project.rows.get(r);
a.add(row.getJSON());
}
} catch (JSONException e) {
respondException(response, e);
}
o.put("start", start);
o.put("limit", limit);
o.put("rows", a);
respondJSON(response, o);
} catch (JSONException e) {
respondException(response, e);
}
}
}

View File

@ -1,6 +1,21 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
public class Cell { import java.io.Serializable;
import org.json.JSONException;
import org.json.JSONObject;
public class Cell implements Serializable {
private static final long serialVersionUID = -5891067829205458102L;
public Object value; public Object value;
public Recon recon; public Recon recon;
public JSONObject getJSON() throws JSONException {
JSONObject o = new JSONObject();
o.put("v", value);
return o;
}
} }

View File

@ -1,7 +1,24 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
public class Column { import java.io.Serializable;
import org.json.JSONException;
import org.json.JSONObject;
public class Column implements Serializable {
private static final long serialVersionUID = -1063342490951563563L;
public int cellIndex; public int cellIndex;
public String headerLabel; public String headerLabel;
public Class valueType; public Class valueType;
public JSONObject getJSON() throws JSONException {
JSONObject o = new JSONObject();
o.put("cellIndex", cellIndex);
o.put("headerLabel", headerLabel);
o.put("valueType", valueType == null ? null : valueType.getSimpleName());
return o;
}
} }

View File

@ -1,8 +1,27 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
public class ColumnModel { import org.json.JSONException;
import org.json.JSONObject;
public class ColumnModel implements Serializable {
private static final long serialVersionUID = 7679639795211544511L;
public List<Column> columns = new LinkedList<Column>(); public List<Column> columns = new LinkedList<Column>();
public JSONObject getJSON() throws JSONException {
JSONObject o = new JSONObject();
List<JSONObject> a = new ArrayList<JSONObject>(columns.size());
for (Column column : columns) {
a.add(column.getJSON());
}
o.put("columns", a);
return o;
}
} }

View File

@ -1,12 +1,12 @@
package com.metaweb.gridlock; package com.metaweb.gridlock.model;
import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.metaweb.gridlock.model.ColumnModel; public class Project implements Serializable {
import com.metaweb.gridlock.model.Row; private static final long serialVersionUID = -5089046824819472570L;
public class Project {
public long id; public long id;
public ColumnModel columnModel = new ColumnModel(); public ColumnModel columnModel = new ColumnModel();
public List<Row> rows = new ArrayList<Row>(); public List<Row> rows = new ArrayList<Row>();

View File

@ -1,11 +1,14 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
import java.io.Serializable;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
public class Recon { public class Recon implements Serializable {
private static final long serialVersionUID = 8906257833709315762L;
static public enum Judgment { static public enum Judgment {
None, None,
Approve, Approve,
@ -14,6 +17,4 @@ public class Recon {
public Map<String, Object> features = new HashMap<String, Object>(); public Map<String, Object> features = new HashMap<String, Object>();
public List<ReconCandidate> candidates = new LinkedList<ReconCandidate>(); public List<ReconCandidate> candidates = new LinkedList<ReconCandidate>();
} }

View File

@ -1,6 +1,10 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
public class ReconCandidate { import java.io.Serializable;
public class ReconCandidate implements Serializable {
private static final long serialVersionUID = -8013997214978715606L;
public String topicID; public String topicID;
public String topicGUID; public String topicGUID;
public String[] typeIDs; public String[] typeIDs;

View File

@ -1,14 +1,34 @@
package com.metaweb.gridlock.model; package com.metaweb.gridlock.model;
import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class Row { import org.json.JSONException;
import org.json.JSONObject;
public class Row implements Serializable {
private static final long serialVersionUID = -689264211730915507L;
public boolean flagged; public boolean flagged;
public boolean starred; public boolean starred;
public List<Cell> cells; public List<Cell> cells;
public Row(int cellCount) { public Row(int cellCount) {
cells = new ArrayList<Cell>(); cells = new ArrayList<Cell>(cellCount);
}
public JSONObject getJSON() throws JSONException {
JSONObject o = new JSONObject();
List<JSONObject> a = new ArrayList<JSONObject>(cells.size());
for (Cell cell : cells) {
a.add(cell.getJSON());
}
o.put("cells", a);
o.put("flagged", flagged);
o.put("starred", starred);
return o;
} }
} }

View File

@ -2,7 +2,7 @@
* Created on Dec 1, 2005 * Created on Dec 1, 2005
* Created by dfhuynh * Created by dfhuynh
*/ */
package com.metaweb.gridlock; package com.metaweb.gridlock.util;
import java.io.IOException; import java.io.IOException;
import java.io.Writer; import java.io.Writer;

View File

@ -1,4 +1,4 @@
package com.metaweb.gridlock; package com.metaweb.gridlock.util;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;

View File

@ -13,6 +13,6 @@
<servlet-mapping> <servlet-mapping>
<servlet-name>Gridlock</servlet-name> <servlet-name>Gridlock</servlet-name>
<url-pattern>/gridlock</url-pattern> <url-pattern>/command/*</url-pattern>
</servlet-mapping> </servlet-mapping>
</web-app> </web-app>

View File

@ -0,0 +1 @@
<html> <head> <title>Gridlock</title> <link rel="stylesheet" href="/styles/common.css" /> <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script> <script type="text/javascript" src="scripts/util/string.js"></script> <script type="text/javascript" src="scripts/index.js"></script> </head> <body> <div id="header"> <h1>Gridlock</h1> </div> <div id="body"> <h2>New Project</h2> <p>Create a new project by uploading a tab-separated value or comma-separated value file.</p> <form id="file-upload-form" method="post" enctype="multipart/form-data" action="/command/create-project-from-upload"> <table cellspacing="10"> <tr><td>Project Name:</td><td><input type="text" size="30" id="project-name-input" name="project-name" /></td></tr> <tr><td>Project Password:</td><td><input type="password" size="30" id="project-password-input" name="project-password" /><br/>optional, not protected, so use some password you don't care to reveal</td></tr> <tr><td>Upload File:</td><td> <input type="file" id="project-file-input" name="project-file" size="50" /> </td></tr> <tr><td></td><td> <input type="submit" value="Create Project" id="upload-file-button" /> </td></tr> </table> </form> </div> </body> </html>

View File

@ -0,0 +1 @@
<html> <head> <title>Gridlock</title> <link rel="stylesheet" href="/styles/common.css" /> <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script> <script type="text/javascript" src="scripts/util/string.js"></script> <script type="text/javascript" src="scripts/project.js"></script> </head> <body> <div id="header"> <h1 id="title">Gridlock</h1> </div> <div id="body"> Loading ... </div> </body> </html>

View File

@ -0,0 +1,13 @@
function onLoad() {
$("#upload-file-button").click(onClickUploadFileButton);
}
$(onLoad);
function onClickUploadFileButton(evt) {
if ($("#project-name-input")[0].value.trim().length == 0) {
window.alert("You must specify a project name.");
evt.preventDefault();
return false;
}
}

View File

@ -0,0 +1,4 @@
function onLoad() {
}
$(onLoad);

View File

@ -0,0 +1,3 @@
String.prototype.trim = function() {
return this.replace(/^\s+/, '').replace(/\s+$/, '');
}

View File

@ -0,0 +1,32 @@
html {
font-size: 62.5%;
font-family: Tahoma, Helvetica, Arial, sans-serif;
}
body {
font-size: 120%;
margin: 0;
padding: 0;
}
table {
font-size: inherit;
}
tr, td {
vertical-align: top;
}
#header {
padding: 10px 20px;
background: #eee;
}
#header h1 {
margin: 0;
}
#body {
padding: 20px;
}

4045
tests/Wpi Data.tsv Normal file

File diff suppressed because it is too large Load Diff