Changed Java packages com.google.gridworks.* to com.google.refine.* and modified other code just enough to start grefine up without error. Much remains to be done. Do not check out the code just yet.
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1288 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh
parent
a40a09b391
commit
edb23eb263
@ -1,243 +0,0 @@
|
||||
package com.google.gridworks.appengine;
|
||||
|
||||
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.allowTruncate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.net.ssl.SSLSession;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpConnectionMetrics;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.ProtocolVersion;
|
||||
import org.apache.http.conn.ManagedClientConnection;
|
||||
import org.apache.http.conn.routing.HttpRoute;
|
||||
import org.apache.http.entity.ByteArrayEntity;
|
||||
import org.apache.http.message.BasicHttpResponse;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
|
||||
import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPMethod;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
|
||||
|
||||
class AppEngineClientConnection implements ManagedClientConnection {
|
||||
// Managed is the composition of ConnectionReleaseTrigger,
|
||||
// HttpClientConnection, HttpConnection, HttpInetConnection
|
||||
|
||||
private HttpRoute _route;
|
||||
private Object _state;
|
||||
private boolean _reuseable;
|
||||
|
||||
public AppEngineClientConnection(HttpRoute route, Object state) {
|
||||
_route = route;
|
||||
_state = state;
|
||||
}
|
||||
|
||||
// ManagedClientConnection methods
|
||||
|
||||
public HttpRoute getRoute() {
|
||||
return _route;
|
||||
}
|
||||
|
||||
public Object getState() {
|
||||
return _state;
|
||||
}
|
||||
|
||||
public SSLSession getSSLSession() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean isSecure() {
|
||||
// XXX maybe parse the url to see if it's https?
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isMarkedReusable() {
|
||||
return _reuseable;
|
||||
}
|
||||
|
||||
public void markReusable() {
|
||||
_reuseable = true;
|
||||
}
|
||||
|
||||
public void layerProtocol(HttpContext context, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void open(HttpRoute route, HttpContext context, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void setIdleDuration(long duration, TimeUnit unit) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void setState(Object state) {
|
||||
_state = state;
|
||||
}
|
||||
|
||||
public void tunnelProxy(HttpHost next, boolean secure, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void tunnelTarget(boolean secure, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void unmarkReusable() {
|
||||
_reuseable = false;
|
||||
}
|
||||
|
||||
|
||||
// ConnectionReleaseTrigger methods
|
||||
|
||||
public void releaseConnection() {
|
||||
return;
|
||||
}
|
||||
|
||||
public void abortConnection() {
|
||||
return;
|
||||
}
|
||||
|
||||
// HttpClientConnection methods
|
||||
|
||||
private HTTPRequest _appengine_hrequest;
|
||||
private HTTPResponse _appengine_hresponse;
|
||||
|
||||
public void flush() {
|
||||
return;
|
||||
}
|
||||
|
||||
public boolean isResponseAvailable(int timeout) {
|
||||
// XXX possibly use Async fetcher
|
||||
return true;
|
||||
}
|
||||
|
||||
public void receiveResponseEntity(org.apache.http.HttpResponse apache_response) {
|
||||
byte[] data = _appengine_hresponse.getContent();
|
||||
|
||||
if (data != null) {
|
||||
apache_response.setEntity(new ByteArrayEntity(data));
|
||||
}
|
||||
}
|
||||
|
||||
public HttpResponse receiveResponseHeader() {
|
||||
URLFetchService ufs = URLFetchServiceFactory.getURLFetchService();
|
||||
try {
|
||||
_appengine_hresponse = ufs.fetch(_appengine_hrequest);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
org.apache.http.HttpResponse apache_response =
|
||||
new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 0),
|
||||
_appengine_hresponse.getResponseCode(),
|
||||
null);
|
||||
|
||||
for (HTTPHeader h : _appengine_hresponse.getHeaders()) {
|
||||
apache_response.addHeader(h.getName(), h.getValue());
|
||||
}
|
||||
|
||||
return apache_response;
|
||||
}
|
||||
|
||||
public void sendRequestEntity(org.apache.http.HttpEntityEnclosingRequest request) {
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
|
||||
org.apache.http.HttpEntity ent = request.getEntity();
|
||||
if (ent != null) {
|
||||
try {
|
||||
ent.writeTo(os);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
_appengine_hrequest.setPayload(os.toByteArray());
|
||||
}
|
||||
|
||||
public void sendRequestHeader(org.apache.http.HttpRequest apache_request) {
|
||||
URL request_url;
|
||||
|
||||
HttpHost host = _route.getTargetHost();
|
||||
|
||||
String protocol = host.getSchemeName();
|
||||
String addr = host.getHostName();
|
||||
int port = host.getPort();
|
||||
|
||||
String path = apache_request.getRequestLine().getUri();
|
||||
|
||||
try {
|
||||
request_url = new URL(protocol, addr, port, path);
|
||||
} catch (java.net.MalformedURLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
HTTPMethod method = HTTPMethod.valueOf(apache_request.getRequestLine().getMethod());
|
||||
_appengine_hrequest = new HTTPRequest(request_url, method, allowTruncate()
|
||||
.doNotFollowRedirects());
|
||||
|
||||
Header[] apache_headers = apache_request.getAllHeaders();
|
||||
for (int i = 0; i < apache_headers.length; i++) {
|
||||
Header h = apache_headers[i];
|
||||
_appengine_hrequest
|
||||
.setHeader(new HTTPHeader(h.getName(), h.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
// HttpConnection methods
|
||||
|
||||
public void close() {
|
||||
return;
|
||||
}
|
||||
|
||||
public HttpConnectionMetrics getMetrics() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getSocketTimeout() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
public boolean isOpen() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean isStale() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public void setSocketTimeout(int timeout) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
return;
|
||||
}
|
||||
|
||||
// HttpInetConnection methods
|
||||
|
||||
public InetAddress getLocalAddress() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getLocalPort() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
public InetAddress getRemoteAddress() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getRemotePort() {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
package com.google.gridworks.appengine;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.http.conn.ClientConnectionManager;
|
||||
import org.apache.http.conn.ClientConnectionRequest;
|
||||
import org.apache.http.conn.ManagedClientConnection;
|
||||
import org.apache.http.conn.routing.HttpRoute;
|
||||
import org.apache.http.conn.scheme.Scheme;
|
||||
import org.apache.http.conn.scheme.SchemeRegistry;
|
||||
import org.apache.http.conn.scheme.SocketFactory;
|
||||
import org.apache.http.params.HttpParams;
|
||||
|
||||
public class AppEngineClientConnectionManager implements ClientConnectionManager {
|
||||
|
||||
private SchemeRegistry schemes;
|
||||
|
||||
class NoopSocketFactory implements SocketFactory {
|
||||
public Socket connectSocket(Socket sock, String host, int port, InetAddress addr, int lport, HttpParams params) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public Socket createSocket() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean isSecure(Socket sock) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public AppEngineClientConnectionManager() {
|
||||
SocketFactory noop_sf = new NoopSocketFactory();
|
||||
schemes = new SchemeRegistry();
|
||||
schemes.register(new Scheme("http", noop_sf, 80));
|
||||
schemes.register(new Scheme("https", noop_sf, 443));
|
||||
}
|
||||
|
||||
public void closeExpiredConnections() {
|
||||
return;
|
||||
}
|
||||
|
||||
public void closeIdleConnections(long idletime, TimeUnit tunit) {
|
||||
return;
|
||||
}
|
||||
|
||||
public ManagedClientConnection getConnection(HttpRoute route, Object state) {
|
||||
return new AppEngineClientConnection(route, state);
|
||||
}
|
||||
|
||||
public SchemeRegistry getSchemeRegistry() {
|
||||
return schemes;
|
||||
}
|
||||
|
||||
public void releaseConnection(ManagedClientConnection conn, long valid, TimeUnit tuint) {
|
||||
return;
|
||||
}
|
||||
|
||||
public ClientConnectionRequest requestConnection(final HttpRoute route, final Object state) {
|
||||
return new ClientConnectionRequest() {
|
||||
public void abortRequest() {
|
||||
return;
|
||||
}
|
||||
|
||||
public ManagedClientConnection getConnection(long idletime, TimeUnit tunit) {
|
||||
return AppEngineClientConnectionManager.this.getConnection(route, state);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -1,373 +0,0 @@
|
||||
package com.google.gridworks.broker;
|
||||
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.jdo.Extent;
|
||||
import javax.jdo.JDOHelper;
|
||||
import javax.jdo.PersistenceManager;
|
||||
import javax.jdo.PersistenceManagerFactory;
|
||||
import javax.jdo.Transaction;
|
||||
import javax.jdo.annotations.IdGeneratorStrategy;
|
||||
import javax.jdo.annotations.PersistenceCapable;
|
||||
import javax.jdo.annotations.Persistent;
|
||||
import javax.jdo.annotations.PrimaryKey;
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.conn.ClientConnectionManager;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.appengine.api.datastore.Text;
|
||||
import com.google.gridworks.appengine.AppEngineClientConnectionManager;
|
||||
import com.google.gridworks.broker.GridworksBroker;
|
||||
|
||||
public class AppEngineGridworksBrokerImpl extends GridworksBroker {
|
||||
|
||||
protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.appengine");
|
||||
|
||||
PersistenceManagerFactory pmfInstance;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
super.init(config);
|
||||
|
||||
pmfInstance = JDOHelper.getPersistenceManagerFactory("transactional");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected HttpClient getHttpClient() {
|
||||
ClientConnectionManager cm = new AppEngineClientConnectionManager();
|
||||
return new DefaultHttpClient(cm, null);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected void expire(HttpServletResponse response) throws Exception {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
protected void expireLocks(HttpServletResponse response) throws Exception {
|
||||
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Extent<Lock> extent = pm.getExtent(Lock.class, false);
|
||||
|
||||
try {
|
||||
for (Lock lock : extent) {
|
||||
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
|
||||
Transaction tx = pm.currentTransaction();
|
||||
try {
|
||||
tx.begin();
|
||||
pm.deletePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extent.closeAll();
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
// TODO: implement
|
||||
respond(response, lockToJSON(getLock(pm,pid)));
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
// TODO: implement
|
||||
|
||||
try {
|
||||
Lock lock = getLock(pm, pid);
|
||||
if (lock == null) {
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
lock = new Lock(Long.toHexString(tx.hashCode()), pid, uid);
|
||||
pm.makePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, lockToJSON(lock));
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
|
||||
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Lock lock = getLock(pm, pid);
|
||||
if (lock != null) {
|
||||
if (!lock.id.equals(lid)) {
|
||||
throw new RuntimeException("Lock id doesn't match, can't release the lock");
|
||||
}
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
pm.deletePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------------------
|
||||
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
checkLock(pm, pid, uid, lid);
|
||||
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
if (project != null) {
|
||||
throw new RuntimeException("Project '" + pid + "' already exists");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
project = new Project(pid, data);
|
||||
pm.makePersistent(project);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
checkLock(pm, pid, uid, lid);
|
||||
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' not found");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
for (String s : transformations) {
|
||||
project.transformations.add(new Text(s));
|
||||
}
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected void openProject(HttpServletResponse response, String pid) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("data"); writer.value(project.data.toString());
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
for (Text s : project.transformations) {
|
||||
writer.value(s.toString());
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
int size = project.transformations.size();
|
||||
for (int i = tindex; i < size; i++) {
|
||||
writer.value(project.transformations.get(i).toString());
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Project getProject(PersistenceManager pm, String pid) {
|
||||
Project project = pm.getObjectById(Project.class, pid);
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' is not managed by this broker");
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
@PersistenceCapable
|
||||
static class Project {
|
||||
|
||||
@PrimaryKey
|
||||
@Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
|
||||
String pid;
|
||||
|
||||
@Persistent
|
||||
List<Text> transformations = new ArrayList<Text>();
|
||||
|
||||
@Persistent
|
||||
Text data;
|
||||
|
||||
Project(String pid, String data) {
|
||||
this.pid = pid;
|
||||
this.data = new Text(data);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Lock getLock(PersistenceManager pm, String pid) {
|
||||
return pm.getObjectById(Lock.class, pid);
|
||||
}
|
||||
|
||||
void checkLock(PersistenceManager pm, String pid, String uid, String lid) {
|
||||
Lock lock = getLock(pm, pid);
|
||||
|
||||
if (lock == null) {
|
||||
throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
|
||||
}
|
||||
|
||||
if (!lock.pid.equals(pid)) {
|
||||
throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid);
|
||||
}
|
||||
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid);
|
||||
}
|
||||
}
|
||||
|
||||
JSONObject lockToJSON(Lock lock) throws JSONException {
|
||||
JSONObject o = new JSONObject();
|
||||
if (lock != null) {
|
||||
o.put("lock_id", lock.id);
|
||||
o.put("project_id", lock.pid);
|
||||
o.put("user_id", lock.uid);
|
||||
o.put("timestamp", lock.timestamp);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
@PersistenceCapable
|
||||
static class Lock {
|
||||
|
||||
@Persistent
|
||||
String id;
|
||||
|
||||
@PrimaryKey
|
||||
@Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
|
||||
String pid;
|
||||
|
||||
@Persistent
|
||||
String uid;
|
||||
|
||||
@Persistent
|
||||
long timestamp;
|
||||
|
||||
Lock(String id, String pid, String uid) {
|
||||
this.id = id;
|
||||
this.pid = pid;
|
||||
this.uid = uid;
|
||||
this.timestamp = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,243 @@
|
||||
package com.google.refine.appengine;
|
||||
|
||||
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.allowTruncate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.net.ssl.SSLSession;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpConnectionMetrics;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.ProtocolVersion;
|
||||
import org.apache.http.conn.ManagedClientConnection;
|
||||
import org.apache.http.conn.routing.HttpRoute;
|
||||
import org.apache.http.entity.ByteArrayEntity;
|
||||
import org.apache.http.message.BasicHttpResponse;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
|
||||
import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPMethod;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
|
||||
|
||||
class AppEngineClientConnection implements ManagedClientConnection {
|
||||
// Managed is the composition of ConnectionReleaseTrigger,
|
||||
// HttpClientConnection, HttpConnection, HttpInetConnection
|
||||
|
||||
private HttpRoute _route;
|
||||
private Object _state;
|
||||
private boolean _reuseable;
|
||||
|
||||
public AppEngineClientConnection(HttpRoute route, Object state) {
|
||||
_route = route;
|
||||
_state = state;
|
||||
}
|
||||
|
||||
// ManagedClientConnection methods
|
||||
|
||||
public HttpRoute getRoute() {
|
||||
return _route;
|
||||
}
|
||||
|
||||
public Object getState() {
|
||||
return _state;
|
||||
}
|
||||
|
||||
public SSLSession getSSLSession() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean isSecure() {
|
||||
// XXX maybe parse the url to see if it's https?
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isMarkedReusable() {
|
||||
return _reuseable;
|
||||
}
|
||||
|
||||
public void markReusable() {
|
||||
_reuseable = true;
|
||||
}
|
||||
|
||||
public void layerProtocol(HttpContext context, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void open(HttpRoute route, HttpContext context, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void setIdleDuration(long duration, TimeUnit unit) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void setState(Object state) {
|
||||
_state = state;
|
||||
}
|
||||
|
||||
public void tunnelProxy(HttpHost next, boolean secure, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void tunnelTarget(boolean secure, HttpParams params) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void unmarkReusable() {
|
||||
_reuseable = false;
|
||||
}
|
||||
|
||||
|
||||
// ConnectionReleaseTrigger methods
|
||||
|
||||
public void releaseConnection() {
|
||||
return;
|
||||
}
|
||||
|
||||
public void abortConnection() {
|
||||
return;
|
||||
}
|
||||
|
||||
// HttpClientConnection methods
|
||||
|
||||
private HTTPRequest _appengine_hrequest;
|
||||
private HTTPResponse _appengine_hresponse;
|
||||
|
||||
public void flush() {
|
||||
return;
|
||||
}
|
||||
|
||||
public boolean isResponseAvailable(int timeout) {
|
||||
// XXX possibly use Async fetcher
|
||||
return true;
|
||||
}
|
||||
|
||||
public void receiveResponseEntity(org.apache.http.HttpResponse apache_response) {
|
||||
byte[] data = _appengine_hresponse.getContent();
|
||||
|
||||
if (data != null) {
|
||||
apache_response.setEntity(new ByteArrayEntity(data));
|
||||
}
|
||||
}
|
||||
|
||||
public HttpResponse receiveResponseHeader() {
|
||||
URLFetchService ufs = URLFetchServiceFactory.getURLFetchService();
|
||||
try {
|
||||
_appengine_hresponse = ufs.fetch(_appengine_hrequest);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
org.apache.http.HttpResponse apache_response =
|
||||
new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 0),
|
||||
_appengine_hresponse.getResponseCode(),
|
||||
null);
|
||||
|
||||
for (HTTPHeader h : _appengine_hresponse.getHeaders()) {
|
||||
apache_response.addHeader(h.getName(), h.getValue());
|
||||
}
|
||||
|
||||
return apache_response;
|
||||
}
|
||||
|
||||
public void sendRequestEntity(org.apache.http.HttpEntityEnclosingRequest request) {
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
|
||||
org.apache.http.HttpEntity ent = request.getEntity();
|
||||
if (ent != null) {
|
||||
try {
|
||||
ent.writeTo(os);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
_appengine_hrequest.setPayload(os.toByteArray());
|
||||
}
|
||||
|
||||
public void sendRequestHeader(org.apache.http.HttpRequest apache_request) {
|
||||
URL request_url;
|
||||
|
||||
HttpHost host = _route.getTargetHost();
|
||||
|
||||
String protocol = host.getSchemeName();
|
||||
String addr = host.getHostName();
|
||||
int port = host.getPort();
|
||||
|
||||
String path = apache_request.getRequestLine().getUri();
|
||||
|
||||
try {
|
||||
request_url = new URL(protocol, addr, port, path);
|
||||
} catch (java.net.MalformedURLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
HTTPMethod method = HTTPMethod.valueOf(apache_request.getRequestLine().getMethod());
|
||||
_appengine_hrequest = new HTTPRequest(request_url, method, allowTruncate()
|
||||
.doNotFollowRedirects());
|
||||
|
||||
Header[] apache_headers = apache_request.getAllHeaders();
|
||||
for (int i = 0; i < apache_headers.length; i++) {
|
||||
Header h = apache_headers[i];
|
||||
_appengine_hrequest
|
||||
.setHeader(new HTTPHeader(h.getName(), h.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
// HttpConnection methods
|
||||
|
||||
public void close() {
|
||||
return;
|
||||
}
|
||||
|
||||
public HttpConnectionMetrics getMetrics() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getSocketTimeout() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
public boolean isOpen() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean isStale() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public void setSocketTimeout(int timeout) {
|
||||
return;
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
return;
|
||||
}
|
||||
|
||||
// HttpInetConnection methods
|
||||
|
||||
public InetAddress getLocalAddress() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getLocalPort() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
public InetAddress getRemoteAddress() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getRemotePort() {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
package com.google.refine.appengine;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.http.conn.ClientConnectionManager;
|
||||
import org.apache.http.conn.ClientConnectionRequest;
|
||||
import org.apache.http.conn.ManagedClientConnection;
|
||||
import org.apache.http.conn.routing.HttpRoute;
|
||||
import org.apache.http.conn.scheme.Scheme;
|
||||
import org.apache.http.conn.scheme.SchemeRegistry;
|
||||
import org.apache.http.conn.scheme.SocketFactory;
|
||||
import org.apache.http.params.HttpParams;
|
||||
|
||||
public class AppEngineClientConnectionManager implements ClientConnectionManager {
|
||||
|
||||
private SchemeRegistry schemes;
|
||||
|
||||
class NoopSocketFactory implements SocketFactory {
|
||||
public Socket connectSocket(Socket sock, String host, int port, InetAddress addr, int lport, HttpParams params) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public Socket createSocket() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean isSecure(Socket sock) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public AppEngineClientConnectionManager() {
|
||||
SocketFactory noop_sf = new NoopSocketFactory();
|
||||
schemes = new SchemeRegistry();
|
||||
schemes.register(new Scheme("http", noop_sf, 80));
|
||||
schemes.register(new Scheme("https", noop_sf, 443));
|
||||
}
|
||||
|
||||
public void closeExpiredConnections() {
|
||||
return;
|
||||
}
|
||||
|
||||
public void closeIdleConnections(long idletime, TimeUnit tunit) {
|
||||
return;
|
||||
}
|
||||
|
||||
public ManagedClientConnection getConnection(HttpRoute route, Object state) {
|
||||
return new AppEngineClientConnection(route, state);
|
||||
}
|
||||
|
||||
public SchemeRegistry getSchemeRegistry() {
|
||||
return schemes;
|
||||
}
|
||||
|
||||
public void releaseConnection(ManagedClientConnection conn, long valid, TimeUnit tuint) {
|
||||
return;
|
||||
}
|
||||
|
||||
public ClientConnectionRequest requestConnection(final HttpRoute route, final Object state) {
|
||||
return new ClientConnectionRequest() {
|
||||
public void abortRequest() {
|
||||
return;
|
||||
}
|
||||
|
||||
public ManagedClientConnection getConnection(long idletime, TimeUnit tunit) {
|
||||
return AppEngineClientConnectionManager.this.getConnection(route, state);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,373 @@
|
||||
package com.google.refine.broker;
|
||||
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.jdo.Extent;
|
||||
import javax.jdo.JDOHelper;
|
||||
import javax.jdo.PersistenceManager;
|
||||
import javax.jdo.PersistenceManagerFactory;
|
||||
import javax.jdo.Transaction;
|
||||
import javax.jdo.annotations.IdGeneratorStrategy;
|
||||
import javax.jdo.annotations.PersistenceCapable;
|
||||
import javax.jdo.annotations.Persistent;
|
||||
import javax.jdo.annotations.PrimaryKey;
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.conn.ClientConnectionManager;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.appengine.api.datastore.Text;
|
||||
import com.google.refine.appengine.AppEngineClientConnectionManager;
|
||||
import com.google.refine.broker.GridworksBroker;
|
||||
|
||||
public class AppEngineGridworksBrokerImpl extends GridworksBroker {
|
||||
|
||||
protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.appengine");
|
||||
|
||||
PersistenceManagerFactory pmfInstance;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
super.init(config);
|
||||
|
||||
pmfInstance = JDOHelper.getPersistenceManagerFactory("transactional");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected HttpClient getHttpClient() {
|
||||
ClientConnectionManager cm = new AppEngineClientConnectionManager();
|
||||
return new DefaultHttpClient(cm, null);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected void expire(HttpServletResponse response) throws Exception {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception {
|
||||
// TODO: implement
|
||||
}
|
||||
|
||||
protected void expireLocks(HttpServletResponse response) throws Exception {
|
||||
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Extent<Lock> extent = pm.getExtent(Lock.class, false);
|
||||
|
||||
try {
|
||||
for (Lock lock : extent) {
|
||||
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
|
||||
Transaction tx = pm.currentTransaction();
|
||||
try {
|
||||
tx.begin();
|
||||
pm.deletePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extent.closeAll();
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
// TODO: implement
|
||||
respond(response, lockToJSON(getLock(pm,pid)));
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
// TODO: implement
|
||||
|
||||
try {
|
||||
Lock lock = getLock(pm, pid);
|
||||
if (lock == null) {
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
lock = new Lock(Long.toHexString(tx.hashCode()), pid, uid);
|
||||
pm.makePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, lockToJSON(lock));
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
|
||||
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Lock lock = getLock(pm, pid);
|
||||
if (lock != null) {
|
||||
if (!lock.id.equals(lid)) {
|
||||
throw new RuntimeException("Lock id doesn't match, can't release the lock");
|
||||
}
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
pm.deletePersistent(lock);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------------------
|
||||
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
checkLock(pm, pid, uid, lid);
|
||||
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
if (project != null) {
|
||||
throw new RuntimeException("Project '" + pid + "' already exists");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
tx.begin();
|
||||
project = new Project(pid, data);
|
||||
pm.makePersistent(project);
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
checkLock(pm, pid, uid, lid);
|
||||
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' not found");
|
||||
}
|
||||
|
||||
Transaction tx = pm.currentTransaction();
|
||||
|
||||
try {
|
||||
for (String s : transformations) {
|
||||
project.transformations.add(new Text(s));
|
||||
}
|
||||
tx.commit();
|
||||
} finally {
|
||||
if (tx.isActive()) {
|
||||
tx.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
protected void openProject(HttpServletResponse response, String pid) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("data"); writer.value(project.data.toString());
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
for (Text s : project.transformations) {
|
||||
writer.value(s.toString());
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception {
|
||||
PersistenceManager pm = pmfInstance.getPersistenceManager();
|
||||
|
||||
try {
|
||||
Project project = getProject(pm, pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
int size = project.transformations.size();
|
||||
for (int i = tindex; i < size; i++) {
|
||||
writer.value(project.transformations.get(i).toString());
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
pm.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Project getProject(PersistenceManager pm, String pid) {
|
||||
Project project = pm.getObjectById(Project.class, pid);
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' is not managed by this broker");
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
@PersistenceCapable
|
||||
static class Project {
|
||||
|
||||
@PrimaryKey
|
||||
@Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
|
||||
String pid;
|
||||
|
||||
@Persistent
|
||||
List<Text> transformations = new ArrayList<Text>();
|
||||
|
||||
@Persistent
|
||||
Text data;
|
||||
|
||||
Project(String pid, String data) {
|
||||
this.pid = pid;
|
||||
this.data = new Text(data);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Lock getLock(PersistenceManager pm, String pid) {
|
||||
return pm.getObjectById(Lock.class, pid);
|
||||
}
|
||||
|
||||
void checkLock(PersistenceManager pm, String pid, String uid, String lid) {
|
||||
Lock lock = getLock(pm, pid);
|
||||
|
||||
if (lock == null) {
|
||||
throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
|
||||
}
|
||||
|
||||
if (!lock.pid.equals(pid)) {
|
||||
throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid);
|
||||
}
|
||||
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid);
|
||||
}
|
||||
}
|
||||
|
||||
JSONObject lockToJSON(Lock lock) throws JSONException {
|
||||
JSONObject o = new JSONObject();
|
||||
if (lock != null) {
|
||||
o.put("lock_id", lock.id);
|
||||
o.put("project_id", lock.pid);
|
||||
o.put("user_id", lock.uid);
|
||||
o.put("timestamp", lock.timestamp);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
@PersistenceCapable
|
||||
static class Lock {
|
||||
|
||||
@Persistent
|
||||
String id;
|
||||
|
||||
@PrimaryKey
|
||||
@Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
|
||||
String pid;
|
||||
|
||||
@Persistent
|
||||
String uid;
|
||||
|
||||
@Persistent
|
||||
long timestamp;
|
||||
|
||||
Lock(String id, String pid, String uid) {
|
||||
this.id = id;
|
||||
this.pid = pid;
|
||||
this.uid = uid;
|
||||
this.timestamp = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,311 +0,0 @@
|
||||
|
||||
package com.google.gridworks.broker;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.ResponseHandler;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.BasicResponseHandler;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.params.CoreProtocolPNames;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import edu.mit.simile.butterfly.ButterflyModuleImpl;
|
||||
|
||||
/**
|
||||
* This class contains all the code shared by various implementations of a Gridworks Broker.
|
||||
*
|
||||
* A broker is a server used by multiple Gridworks installations to enable collaborative
|
||||
* development over the same project.
|
||||
*
|
||||
* Broker implementations differ in how they store their state but all of them are required
|
||||
* to extend this abstract class and implement the services that are called via HTTP.
|
||||
*
|
||||
*/
|
||||
public abstract class GridworksBroker extends ButterflyModuleImpl {
|
||||
|
||||
static final public String GET_STATE = "get_state";
|
||||
static final public String EXPIRE = "expire";
|
||||
static final public String OBTAIN_LOCK = "obtain_lock";
|
||||
static final public String RELEASE_LOCK = "release_lock";
|
||||
static final public String TRANSFORM = "transform";
|
||||
static final public String START = "start";
|
||||
static final public String OPEN = "open";
|
||||
|
||||
static final public int ALL = 0;
|
||||
static final public int COL = 1;
|
||||
static final public int CELL = 2;
|
||||
|
||||
static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker");
|
||||
|
||||
static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info";
|
||||
static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials";
|
||||
static final protected String OAUTH_HEADER = "Authorization";
|
||||
|
||||
static protected String OK;
|
||||
|
||||
static {
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status","ok");
|
||||
OK = o.toString();
|
||||
} catch (JSONException e) {
|
||||
// not going to happen;
|
||||
}
|
||||
}
|
||||
|
||||
static public final long LOCK_DURATION = 60 * 1000; // 1 minute
|
||||
static public final long USER_DURATION = 5 * 60 * 1000; // 1 minute
|
||||
static public final long LOCK_EXPIRATION_CHECK_DELAY = 5 * 1000; // 5 seconds
|
||||
|
||||
protected HttpClient httpclient;
|
||||
|
||||
protected boolean developmentMode;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
super.init(config);
|
||||
httpclient = getHttpClient();
|
||||
developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
|
||||
if (developmentMode) logger.warn("Running in development mode");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
httpclient.getConnectionManager().shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean process(String path, HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("> process '{}'", path);
|
||||
} else {
|
||||
logger.info("process '{}'", path);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
if (GET_STATE.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
getState(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "rev"));
|
||||
} else if (EXPIRE.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
expire(response);
|
||||
} else if (OBTAIN_LOCK.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
obtainLock(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "locktype"), getParameter(request, "lockvalue"));
|
||||
} else if (RELEASE_LOCK.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
releaseLock(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"));
|
||||
} else if (TRANSFORM.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
addTransformations(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getList(request, "transformations"));
|
||||
} else if (START.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
startProject(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getData(request), getParameter(request, "metadata"), getList(request, "transformations"));
|
||||
} else if (OPEN.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
openProject(response, getParameter(request, "pid"));
|
||||
} else {
|
||||
boolean value = super.process(path, request, response);
|
||||
if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
|
||||
return value;
|
||||
}
|
||||
|
||||
} catch (RuntimeException e) {
|
||||
logger.error("runtime error", e.getMessage());
|
||||
respondError(response, e.getMessage());
|
||||
} catch (Exception e) {
|
||||
logger.error("internal error", e);
|
||||
respondException(response, e);
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
protected abstract HttpClient getHttpClient();
|
||||
|
||||
protected abstract void expire(HttpServletResponse response) throws Exception;
|
||||
|
||||
protected abstract void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception;
|
||||
|
||||
protected abstract void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception;
|
||||
|
||||
protected abstract void releaseLock(HttpServletResponse response, String pid, String uid, String lock) throws Exception;
|
||||
|
||||
protected abstract void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception;
|
||||
|
||||
protected abstract void addTransformations(HttpServletResponse response, String pid, String uid, String lock, List<String> transformations) throws Exception;
|
||||
|
||||
protected abstract void openProject(HttpServletResponse response, String pid) throws Exception;
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected String getUserId(HttpServletRequest request) throws Exception {
|
||||
|
||||
// This is useful for testing
|
||||
if (developmentMode) {
|
||||
return getParameter(request, "uid");
|
||||
}
|
||||
|
||||
String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
|
||||
if (oauth == null) {
|
||||
throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
|
||||
}
|
||||
|
||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||
Map<String,String> params = (Map<String,String>) request.getParameterMap();
|
||||
for (Entry<String,String> e : params.entrySet()) {
|
||||
formparams.add(new BasicNameValuePair((String) e.getKey(), (String) e.getValue()));
|
||||
}
|
||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||
|
||||
HttpPost httpRequest = new HttpPost(USER_INFO_URL);
|
||||
httpRequest.setHeader(OAUTH_HEADER, oauth);
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks Broker");
|
||||
httpRequest.setEntity(entity);
|
||||
|
||||
ResponseHandler<String> responseHandler = new BasicResponseHandler();
|
||||
String responseBody = httpclient.execute(httpRequest, responseHandler);
|
||||
JSONObject o = new JSONObject(responseBody);
|
||||
|
||||
return o.getString("username");
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
|
||||
String param = request.getParameter(name);
|
||||
if (param == null) {
|
||||
throw new RuntimeException("request must come with a '" + name + "' parameter");
|
||||
}
|
||||
return param;
|
||||
}
|
||||
|
||||
static protected List<String> getList(HttpServletRequest request, String name) throws ServletException, JSONException {
|
||||
String param = getParameter(request, name);
|
||||
JSONArray a = new JSONArray(param);
|
||||
List<String> result = new ArrayList<String>(a.length());
|
||||
for (int i = 0; i < a.length(); i++) {
|
||||
result.add(a.getString(i));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
|
||||
return Integer.parseInt(getParameter(request, name));
|
||||
}
|
||||
|
||||
static protected byte[] getData(HttpServletRequest request) throws ServletException, IOException {
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
InputStream input = request.getInputStream();
|
||||
byte[] buffer = new byte[4096];
|
||||
int count = 0;
|
||||
int n = 0;
|
||||
while (-1 != (n = input.read(buffer))) {
|
||||
output.write(buffer, 0, n);
|
||||
count += n;
|
||||
}
|
||||
return output.toByteArray();
|
||||
|
||||
}
|
||||
|
||||
static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
|
||||
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status", "error");
|
||||
o.put("message", error);
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
respond(response, o.toString());
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace(response.getWriter());
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respondException(HttpServletResponse response, Exception e) throws IOException, ServletException {
|
||||
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status", "error");
|
||||
o.put("message", e.getMessage());
|
||||
|
||||
StringWriter sw = new StringWriter();
|
||||
PrintWriter pw = new PrintWriter(sw);
|
||||
e.printStackTrace(pw);
|
||||
pw.flush();
|
||||
sw.flush();
|
||||
|
||||
o.put("stack", sw.toString());
|
||||
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
respond(response, o.toString());
|
||||
} catch (JSONException e1) {
|
||||
e.printStackTrace(response.getWriter());
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, JSONObject content) throws IOException, ServletException {
|
||||
if (content == null) {
|
||||
throw new ServletException("Content object can't be null");
|
||||
}
|
||||
|
||||
respond(response, content.toString());
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, String content) throws IOException, ServletException {
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
Writer w = response.getWriter();
|
||||
if (w != null) {
|
||||
w.write(content);
|
||||
w.flush();
|
||||
w.close();
|
||||
} else {
|
||||
throw new ServletException("response returned a null writer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,592 +0,0 @@
|
||||
package com.google.gridworks.broker;
|
||||
|
||||
import static com.sleepycat.persist.model.Relationship.MANY_TO_ONE;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Timer;
|
||||
import java.util.TimerTask;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.sleepycat.je.Environment;
|
||||
import com.sleepycat.je.EnvironmentConfig;
|
||||
import com.sleepycat.je.Transaction;
|
||||
import com.sleepycat.persist.EntityCursor;
|
||||
import com.sleepycat.persist.EntityStore;
|
||||
import com.sleepycat.persist.PrimaryIndex;
|
||||
import com.sleepycat.persist.SecondaryIndex;
|
||||
import com.sleepycat.persist.StoreConfig;
|
||||
import com.sleepycat.persist.model.Entity;
|
||||
import com.sleepycat.persist.model.PrimaryKey;
|
||||
import com.sleepycat.persist.model.SecondaryKey;
|
||||
|
||||
public class GridworksBrokerImpl extends GridworksBroker {
|
||||
|
||||
protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.local");
|
||||
|
||||
Environment env;
|
||||
|
||||
EntityStore projectStore;
|
||||
EntityStore lockStore;
|
||||
EntityStore userStore;
|
||||
|
||||
PrimaryIndex<String,Project> projectById;
|
||||
PrimaryIndex<String,Lock> lockById;
|
||||
|
||||
SecondaryIndex<String,String,Lock> locksByProject;
|
||||
|
||||
Timer timer;
|
||||
Expirer expirer;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
logger.trace("> init");
|
||||
super.init(config);
|
||||
|
||||
timer = new Timer();
|
||||
expirer = new Expirer();
|
||||
timer.schedule(expirer, 0, LOCK_EXPIRATION_CHECK_DELAY);
|
||||
|
||||
String dataDir = config.getInitParameter("gridworks.data");
|
||||
if (dataDir == null) dataDir = "data";
|
||||
File dataPath = new File(dataDir);
|
||||
if (!dataPath.exists()) dataPath.mkdirs();
|
||||
|
||||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||||
envConfig.setAllowCreate(true);
|
||||
envConfig.setTransactional(true);
|
||||
env = new Environment(dataPath, envConfig);
|
||||
|
||||
StoreConfig storeConfig = new StoreConfig();
|
||||
storeConfig.setAllowCreate(true);
|
||||
storeConfig.setTransactional(true);
|
||||
projectStore = new EntityStore(env, "ProjectsStore", storeConfig);
|
||||
lockStore = new EntityStore(env, "LockStore", storeConfig);
|
||||
|
||||
projectById = projectStore.getPrimaryIndex(String.class, Project.class);
|
||||
lockById = lockStore.getPrimaryIndex(String.class, Lock.class);
|
||||
|
||||
locksByProject = lockStore.getSecondaryIndex(lockById, String.class, "pid");
|
||||
logger.trace("< init");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
logger.trace("> destroy");
|
||||
super.destroy();
|
||||
|
||||
if (projectStore != null) {
|
||||
projectStore.close();
|
||||
projectById = null;
|
||||
}
|
||||
|
||||
if (lockStore != null) {
|
||||
lockStore.close();
|
||||
lockById = null;
|
||||
}
|
||||
|
||||
if (timer != null) {
|
||||
timer.cancel();
|
||||
timer.purge();
|
||||
timer = null;
|
||||
}
|
||||
|
||||
if (env != null) {
|
||||
env.close();
|
||||
env = null;
|
||||
}
|
||||
logger.trace("< destroy");
|
||||
}
|
||||
|
||||
class Expirer extends TimerTask {
|
||||
public void run() {
|
||||
if (lockById != null) {
|
||||
logger.trace("> expire");
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
try {
|
||||
EntityCursor<Lock> cursor = lockById.entities();
|
||||
try {
|
||||
for (Lock lock : cursor) {
|
||||
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
|
||||
logger.trace("Found expired lock {}", lock.id);
|
||||
try {
|
||||
releaseLock(null, lock.pid, lock.uid, lock.id);
|
||||
} catch (Exception e) {
|
||||
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
logger.trace("< expire");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected HttpClient getHttpClient() {
|
||||
return new DefaultHttpClient();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void expire(HttpServletResponse response) throws Exception {
|
||||
expirer.run();
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
|
||||
logger.trace("> obtain lock");
|
||||
Lock lock = null;
|
||||
Lock blocker = null;
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
/*
|
||||
* ALL
|
||||
* blocked -> somebody else's lock
|
||||
* reuse -> you already have an ALL lock
|
||||
* new -> else
|
||||
*
|
||||
* COL
|
||||
* blocked -> somebody else's all lock || a lock on the same col
|
||||
* reuse -> you have an ALL lock || a lock on the same col
|
||||
* new -> else
|
||||
*
|
||||
* CELL
|
||||
* blocked -> somebody else's all lock || a lock on the same col || a lock on the same cell
|
||||
* reuse -> you have a lock on the same cell
|
||||
* yes -> (you have a lock on the same cell) && (nobody else has a lock on the same cell || the same col || all)
|
||||
* new -> else
|
||||
*
|
||||
*/
|
||||
|
||||
try {
|
||||
if (locktype == ALL) {
|
||||
if (lockvalue.length() > 0) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling an ALL with a specific value, are you sure you didn't want another type of lock?");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
blocker = l;
|
||||
break;
|
||||
} else {
|
||||
if (l.type == ALL) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (locktype == COL) {
|
||||
if (lockvalue.indexOf(',') > -1) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling a COL lock with a CELL value");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue)) ||
|
||||
(l.type == CELL && l.value.split(",")[0].equals(lockvalue))) {
|
||||
blocker = l;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue))) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (locktype == CELL) {
|
||||
if (lockvalue.indexOf(',') == -1) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling a CELL lock without specifying row and column: format must be 'row,column'");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue.split(",")[0])) ||
|
||||
(l.type == CELL && l.value.equals(lockvalue))) {
|
||||
blocker = l;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue.split(",")[0])) ||
|
||||
(l.type == CELL && l.value.equals(lockvalue))) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
|
||||
if (blocker != null) {
|
||||
logger.info("found a blocking lock {}", lockToString(blocker));
|
||||
throw new RuntimeException("Can't obtain lock, it is blocked by a type '" + blocker.type + "' lock owned by '" + blocker.uid + "'");
|
||||
}
|
||||
|
||||
if (lock == null) {
|
||||
logger.info("no comparable lock already exists, creating a new one");
|
||||
lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue);
|
||||
lockById.put(txn, lock);
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
JSONObject o = lockToJSON(lock, uid);
|
||||
o.put("status", "ok");
|
||||
respond(response, o);
|
||||
|
||||
logger.trace("< obtain lock");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
if (lock != null) {
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
|
||||
}
|
||||
lockById.delete(lid);
|
||||
txn.commit();
|
||||
}
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (response != null) { // this because the expiration thread can call this method without a real response
|
||||
respond(response, OK);
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lid, byte[] data, String metadata, List<String> transformations) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
if (projectById.contains(pid)) {
|
||||
throw new RuntimeException("Project '" + pid + "' already exists");
|
||||
}
|
||||
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
|
||||
if (lock.type != ALL) {
|
||||
throw new RuntimeException("The lock you have is not enough to start a project");
|
||||
}
|
||||
|
||||
projectById.put(txn, new Project(pid, data, metadata, transformations));
|
||||
txn.commit();
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
Project project = getProject(pid);
|
||||
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' not found");
|
||||
}
|
||||
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
|
||||
logger.info("obtained lock: {}", lockToString(lock));
|
||||
|
||||
if (lock.type == ALL) {
|
||||
project.transformations.addAll(transformations);
|
||||
} else {
|
||||
for (String s : transformations) {
|
||||
JSONObject o = new JSONObject(s);
|
||||
|
||||
int type = o.getInt("op_type");
|
||||
String value = o.getString("op_value");
|
||||
if (lock.type == COL) {
|
||||
if (type == COL) {
|
||||
if (value != null && value.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify column '" + value + "'.");
|
||||
}
|
||||
} else if (type == CELL) {
|
||||
String column = value.split(",")[0];
|
||||
if (column != null && column.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify cell '" + value + "' in another column.");
|
||||
}
|
||||
}
|
||||
} else if (lock.type == CELL) {
|
||||
if (type == COL) {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column.");
|
||||
} else if (type == CELL) {
|
||||
if (value != null && value.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for cell '" + lock.value + "' and you're attempting to modify cell '" + value + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
projectById.put(txn, project);
|
||||
|
||||
txn.commit();
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void openProject(HttpServletResponse response, String pid) throws Exception {
|
||||
Project project = getProject(pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("status"); writer.value("ok");
|
||||
writer.key("data"); writer.value(project.data);
|
||||
writer.key("metadata"); writer.value(new JSONObject(project.metadata));
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
for (String s : project.transformations) {
|
||||
writer.value(new JSONObject(s));
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
|
||||
|
||||
Project project = getProject(pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
|
||||
writer.object();
|
||||
writer.key("status"); writer.value("ok");
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
int size = project.transformations.size();
|
||||
for (int i = rev; i < size; i++) {
|
||||
writer.value(new JSONObject(project.transformations.get(i)));
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
try {
|
||||
writer.key("locks");
|
||||
writer.array();
|
||||
for (Lock lock : cursor) {
|
||||
writer.value(lockToJSON(lock, uid));
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Project getProject(String pid) {
|
||||
Project project = projectById.get(pid);
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' could not be found: are you sure is not managed by another broker?");
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
@Entity
|
||||
static class Project {
|
||||
|
||||
@PrimaryKey
|
||||
String pid;
|
||||
|
||||
List<String> transformations;
|
||||
|
||||
byte[] data;
|
||||
|
||||
String metadata;
|
||||
|
||||
int rev;
|
||||
|
||||
Project(String pid, byte[] data, String metadata, List<String> transformations) {
|
||||
this.pid = pid;
|
||||
this.data = data;
|
||||
this.metadata = metadata;
|
||||
this.transformations = (transformations != null) ? transformations : new ArrayList<String>();
|
||||
this.rev = this.transformations.size();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private Project() {}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Lock getLock(String lid, String pid, String uid) {
|
||||
Lock lock = lockById.get(lid);
|
||||
checkLock(lock, lid, pid, uid);
|
||||
return lock;
|
||||
}
|
||||
|
||||
void checkLock(Lock lock, String lid, String pid, String uid) {
|
||||
if (lock == null) {
|
||||
throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
|
||||
}
|
||||
|
||||
if (!lock.pid.equals(pid)) {
|
||||
throw new RuntimeException("Lock '" + lock.id + "' is for another project: " + lock.pid);
|
||||
}
|
||||
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("Lock '" + lock.id + "' is owned by another user: " + lock.uid);
|
||||
}
|
||||
}
|
||||
|
||||
Lock getLock(String pid, String uid, int locktype) {
|
||||
Lock lock = null;
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
try {
|
||||
for (Lock l : cursor) {
|
||||
if (uid.equals(l.uid) && (locktype == l.type)) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
|
||||
return lock;
|
||||
}
|
||||
|
||||
JSONObject lockToJSON(Lock lock, String uid) throws JSONException {
|
||||
JSONObject o = new JSONObject();
|
||||
if (lock != null) {
|
||||
// NOTE: only the owner of the lock should get the ID,
|
||||
// otherwise others can just fake ownership of other people's locks
|
||||
if (lock.uid.equals(uid)) {
|
||||
o.put("lock", lock.id);
|
||||
}
|
||||
|
||||
o.put("pid", lock.pid);
|
||||
o.put("uid", lock.uid);
|
||||
o.put("type", lock.type);
|
||||
o.put("value", lock.value);
|
||||
o.put("timestamp", lock.timestamp);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
String lockToString(Lock lock) {
|
||||
return lock.id + "," + lock.pid + "," + lock.uid + "," + lock.type + "," + lock.value;
|
||||
}
|
||||
|
||||
@Entity
|
||||
static class Lock {
|
||||
|
||||
@PrimaryKey
|
||||
String id;
|
||||
|
||||
@SecondaryKey(relate=MANY_TO_ONE)
|
||||
String pid;
|
||||
|
||||
String uid;
|
||||
|
||||
int type;
|
||||
|
||||
String value;
|
||||
|
||||
long timestamp;
|
||||
|
||||
Lock(String id, String pid, String uid, int type, String value) {
|
||||
this.id = id;
|
||||
this.pid = pid;
|
||||
this.uid = uid;
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
this.timestamp = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private Lock() {}
|
||||
}
|
||||
}
|
||||
311
broker/core/src/com/google/refine/broker/GridworksBroker.java
Normal file
311
broker/core/src/com/google/refine/broker/GridworksBroker.java
Normal file
@ -0,0 +1,311 @@
|
||||
|
||||
package com.google.refine.broker;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.ResponseHandler;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.BasicResponseHandler;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.params.CoreProtocolPNames;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import edu.mit.simile.butterfly.ButterflyModuleImpl;
|
||||
|
||||
/**
|
||||
* This class contains all the code shared by various implementations of a Gridworks Broker.
|
||||
*
|
||||
* A broker is a server used by multiple Gridworks installations to enable collaborative
|
||||
* development over the same project.
|
||||
*
|
||||
* Broker implementations differ in how they store their state but all of them are required
|
||||
* to extend this abstract class and implement the services that are called via HTTP.
|
||||
*
|
||||
*/
|
||||
public abstract class GridworksBroker extends ButterflyModuleImpl {
|
||||
|
||||
static final public String GET_STATE = "get_state";
|
||||
static final public String EXPIRE = "expire";
|
||||
static final public String OBTAIN_LOCK = "obtain_lock";
|
||||
static final public String RELEASE_LOCK = "release_lock";
|
||||
static final public String TRANSFORM = "transform";
|
||||
static final public String START = "start";
|
||||
static final public String OPEN = "open";
|
||||
|
||||
static final public int ALL = 0;
|
||||
static final public int COL = 1;
|
||||
static final public int CELL = 2;
|
||||
|
||||
static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker");
|
||||
|
||||
static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info";
|
||||
static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials";
|
||||
static final protected String OAUTH_HEADER = "Authorization";
|
||||
|
||||
static protected String OK;
|
||||
|
||||
static {
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status","ok");
|
||||
OK = o.toString();
|
||||
} catch (JSONException e) {
|
||||
// not going to happen;
|
||||
}
|
||||
}
|
||||
|
||||
static public final long LOCK_DURATION = 60 * 1000; // 1 minute
|
||||
static public final long USER_DURATION = 5 * 60 * 1000; // 1 minute
|
||||
static public final long LOCK_EXPIRATION_CHECK_DELAY = 5 * 1000; // 5 seconds
|
||||
|
||||
protected HttpClient httpclient;
|
||||
|
||||
protected boolean developmentMode;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
super.init(config);
|
||||
httpclient = getHttpClient();
|
||||
developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
|
||||
if (developmentMode) logger.warn("Running in development mode");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
httpclient.getConnectionManager().shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean process(String path, HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("> process '{}'", path);
|
||||
} else {
|
||||
logger.info("process '{}'", path);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
if (GET_STATE.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
getState(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "rev"));
|
||||
} else if (EXPIRE.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
expire(response);
|
||||
} else if (OBTAIN_LOCK.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
obtainLock(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "locktype"), getParameter(request, "lockvalue"));
|
||||
} else if (RELEASE_LOCK.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
releaseLock(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"));
|
||||
} else if (TRANSFORM.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
addTransformations(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getList(request, "transformations"));
|
||||
} else if (START.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
startProject(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getData(request), getParameter(request, "metadata"), getList(request, "transformations"));
|
||||
} else if (OPEN.equals(path)) {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
openProject(response, getParameter(request, "pid"));
|
||||
} else {
|
||||
boolean value = super.process(path, request, response);
|
||||
if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
|
||||
return value;
|
||||
}
|
||||
|
||||
} catch (RuntimeException e) {
|
||||
logger.error("runtime error", e.getMessage());
|
||||
respondError(response, e.getMessage());
|
||||
} catch (Exception e) {
|
||||
logger.error("internal error", e);
|
||||
respondException(response, e);
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
protected abstract HttpClient getHttpClient();
|
||||
|
||||
protected abstract void expire(HttpServletResponse response) throws Exception;
|
||||
|
||||
protected abstract void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception;
|
||||
|
||||
protected abstract void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception;
|
||||
|
||||
protected abstract void releaseLock(HttpServletResponse response, String pid, String uid, String lock) throws Exception;
|
||||
|
||||
protected abstract void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception;
|
||||
|
||||
protected abstract void addTransformations(HttpServletResponse response, String pid, String uid, String lock, List<String> transformations) throws Exception;
|
||||
|
||||
protected abstract void openProject(HttpServletResponse response, String pid) throws Exception;
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected String getUserId(HttpServletRequest request) throws Exception {
|
||||
|
||||
// This is useful for testing
|
||||
if (developmentMode) {
|
||||
return getParameter(request, "uid");
|
||||
}
|
||||
|
||||
String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
|
||||
if (oauth == null) {
|
||||
throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
|
||||
}
|
||||
|
||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||
Map<String,String> params = (Map<String,String>) request.getParameterMap();
|
||||
for (Entry<String,String> e : params.entrySet()) {
|
||||
formparams.add(new BasicNameValuePair((String) e.getKey(), (String) e.getValue()));
|
||||
}
|
||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||
|
||||
HttpPost httpRequest = new HttpPost(USER_INFO_URL);
|
||||
httpRequest.setHeader(OAUTH_HEADER, oauth);
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks Broker");
|
||||
httpRequest.setEntity(entity);
|
||||
|
||||
ResponseHandler<String> responseHandler = new BasicResponseHandler();
|
||||
String responseBody = httpclient.execute(httpRequest, responseHandler);
|
||||
JSONObject o = new JSONObject(responseBody);
|
||||
|
||||
return o.getString("username");
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
|
||||
String param = request.getParameter(name);
|
||||
if (param == null) {
|
||||
throw new RuntimeException("request must come with a '" + name + "' parameter");
|
||||
}
|
||||
return param;
|
||||
}
|
||||
|
||||
static protected List<String> getList(HttpServletRequest request, String name) throws ServletException, JSONException {
|
||||
String param = getParameter(request, name);
|
||||
JSONArray a = new JSONArray(param);
|
||||
List<String> result = new ArrayList<String>(a.length());
|
||||
for (int i = 0; i < a.length(); i++) {
|
||||
result.add(a.getString(i));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
|
||||
return Integer.parseInt(getParameter(request, name));
|
||||
}
|
||||
|
||||
static protected byte[] getData(HttpServletRequest request) throws ServletException, IOException {
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
InputStream input = request.getInputStream();
|
||||
byte[] buffer = new byte[4096];
|
||||
int count = 0;
|
||||
int n = 0;
|
||||
while (-1 != (n = input.read(buffer))) {
|
||||
output.write(buffer, 0, n);
|
||||
count += n;
|
||||
}
|
||||
return output.toByteArray();
|
||||
|
||||
}
|
||||
|
||||
static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
|
||||
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status", "error");
|
||||
o.put("message", error);
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
respond(response, o.toString());
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace(response.getWriter());
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respondException(HttpServletResponse response, Exception e) throws IOException, ServletException {
|
||||
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("status", "error");
|
||||
o.put("message", e.getMessage());
|
||||
|
||||
StringWriter sw = new StringWriter();
|
||||
PrintWriter pw = new PrintWriter(sw);
|
||||
e.printStackTrace(pw);
|
||||
pw.flush();
|
||||
sw.flush();
|
||||
|
||||
o.put("stack", sw.toString());
|
||||
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
respond(response, o.toString());
|
||||
} catch (JSONException e1) {
|
||||
e.printStackTrace(response.getWriter());
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, JSONObject content) throws IOException, ServletException {
|
||||
if (content == null) {
|
||||
throw new ServletException("Content object can't be null");
|
||||
}
|
||||
|
||||
respond(response, content.toString());
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, String content) throws IOException, ServletException {
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
Writer w = response.getWriter();
|
||||
if (w != null) {
|
||||
w.write(content);
|
||||
w.flush();
|
||||
w.close();
|
||||
} else {
|
||||
throw new ServletException("response returned a null writer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,592 @@
|
||||
package com.google.refine.broker;
|
||||
|
||||
import static com.sleepycat.persist.model.Relationship.MANY_TO_ONE;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Timer;
|
||||
import java.util.TimerTask;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.sleepycat.je.Environment;
|
||||
import com.sleepycat.je.EnvironmentConfig;
|
||||
import com.sleepycat.je.Transaction;
|
||||
import com.sleepycat.persist.EntityCursor;
|
||||
import com.sleepycat.persist.EntityStore;
|
||||
import com.sleepycat.persist.PrimaryIndex;
|
||||
import com.sleepycat.persist.SecondaryIndex;
|
||||
import com.sleepycat.persist.StoreConfig;
|
||||
import com.sleepycat.persist.model.Entity;
|
||||
import com.sleepycat.persist.model.PrimaryKey;
|
||||
import com.sleepycat.persist.model.SecondaryKey;
|
||||
|
||||
public class GridworksBrokerImpl extends GridworksBroker {
|
||||
|
||||
protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.local");
|
||||
|
||||
Environment env;
|
||||
|
||||
EntityStore projectStore;
|
||||
EntityStore lockStore;
|
||||
EntityStore userStore;
|
||||
|
||||
PrimaryIndex<String,Project> projectById;
|
||||
PrimaryIndex<String,Lock> lockById;
|
||||
|
||||
SecondaryIndex<String,String,Lock> locksByProject;
|
||||
|
||||
Timer timer;
|
||||
Expirer expirer;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws Exception {
|
||||
logger.trace("> init");
|
||||
super.init(config);
|
||||
|
||||
timer = new Timer();
|
||||
expirer = new Expirer();
|
||||
timer.schedule(expirer, 0, LOCK_EXPIRATION_CHECK_DELAY);
|
||||
|
||||
String dataDir = config.getInitParameter("gridworks.data");
|
||||
if (dataDir == null) dataDir = "data";
|
||||
File dataPath = new File(dataDir);
|
||||
if (!dataPath.exists()) dataPath.mkdirs();
|
||||
|
||||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||||
envConfig.setAllowCreate(true);
|
||||
envConfig.setTransactional(true);
|
||||
env = new Environment(dataPath, envConfig);
|
||||
|
||||
StoreConfig storeConfig = new StoreConfig();
|
||||
storeConfig.setAllowCreate(true);
|
||||
storeConfig.setTransactional(true);
|
||||
projectStore = new EntityStore(env, "ProjectsStore", storeConfig);
|
||||
lockStore = new EntityStore(env, "LockStore", storeConfig);
|
||||
|
||||
projectById = projectStore.getPrimaryIndex(String.class, Project.class);
|
||||
lockById = lockStore.getPrimaryIndex(String.class, Lock.class);
|
||||
|
||||
locksByProject = lockStore.getSecondaryIndex(lockById, String.class, "pid");
|
||||
logger.trace("< init");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
logger.trace("> destroy");
|
||||
super.destroy();
|
||||
|
||||
if (projectStore != null) {
|
||||
projectStore.close();
|
||||
projectById = null;
|
||||
}
|
||||
|
||||
if (lockStore != null) {
|
||||
lockStore.close();
|
||||
lockById = null;
|
||||
}
|
||||
|
||||
if (timer != null) {
|
||||
timer.cancel();
|
||||
timer.purge();
|
||||
timer = null;
|
||||
}
|
||||
|
||||
if (env != null) {
|
||||
env.close();
|
||||
env = null;
|
||||
}
|
||||
logger.trace("< destroy");
|
||||
}
|
||||
|
||||
class Expirer extends TimerTask {
|
||||
public void run() {
|
||||
if (lockById != null) {
|
||||
logger.trace("> expire");
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
try {
|
||||
EntityCursor<Lock> cursor = lockById.entities();
|
||||
try {
|
||||
for (Lock lock : cursor) {
|
||||
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
|
||||
logger.trace("Found expired lock {}", lock.id);
|
||||
try {
|
||||
releaseLock(null, lock.pid, lock.uid, lock.id);
|
||||
} catch (Exception e) {
|
||||
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
logger.trace("< expire");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected HttpClient getHttpClient() {
|
||||
return new DefaultHttpClient();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void expire(HttpServletResponse response) throws Exception {
|
||||
expirer.run();
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
|
||||
logger.trace("> obtain lock");
|
||||
Lock lock = null;
|
||||
Lock blocker = null;
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
/*
|
||||
* ALL
|
||||
* blocked -> somebody else's lock
|
||||
* reuse -> you already have an ALL lock
|
||||
* new -> else
|
||||
*
|
||||
* COL
|
||||
* blocked -> somebody else's all lock || a lock on the same col
|
||||
* reuse -> you have an ALL lock || a lock on the same col
|
||||
* new -> else
|
||||
*
|
||||
* CELL
|
||||
* blocked -> somebody else's all lock || a lock on the same col || a lock on the same cell
|
||||
* reuse -> you have a lock on the same cell
|
||||
* yes -> (you have a lock on the same cell) && (nobody else has a lock on the same cell || the same col || all)
|
||||
* new -> else
|
||||
*
|
||||
*/
|
||||
|
||||
try {
|
||||
if (locktype == ALL) {
|
||||
if (lockvalue.length() > 0) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling an ALL with a specific value, are you sure you didn't want another type of lock?");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
blocker = l;
|
||||
break;
|
||||
} else {
|
||||
if (l.type == ALL) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (locktype == COL) {
|
||||
if (lockvalue.indexOf(',') > -1) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling a COL lock with a CELL value");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue)) ||
|
||||
(l.type == CELL && l.value.split(",")[0].equals(lockvalue))) {
|
||||
blocker = l;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue))) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (locktype == CELL) {
|
||||
if (lockvalue.indexOf(',') == -1) {
|
||||
throw new RuntimeException("Hmm, seems like you're calling a CELL lock without specifying row and column: format must be 'row,column'");
|
||||
}
|
||||
|
||||
for (Lock l : cursor) {
|
||||
if (!l.uid.equals(uid)) {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue.split(",")[0])) ||
|
||||
(l.type == CELL && l.value.equals(lockvalue))) {
|
||||
blocker = l;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (l.type == ALL ||
|
||||
(l.type == COL && l.value.equals(lockvalue.split(",")[0])) ||
|
||||
(l.type == CELL && l.value.equals(lockvalue))) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
|
||||
if (blocker != null) {
|
||||
logger.info("found a blocking lock {}", lockToString(blocker));
|
||||
throw new RuntimeException("Can't obtain lock, it is blocked by a type '" + blocker.type + "' lock owned by '" + blocker.uid + "'");
|
||||
}
|
||||
|
||||
if (lock == null) {
|
||||
logger.info("no comparable lock already exists, creating a new one");
|
||||
lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue);
|
||||
lockById.put(txn, lock);
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
JSONObject o = lockToJSON(lock, uid);
|
||||
o.put("status", "ok");
|
||||
respond(response, o);
|
||||
|
||||
logger.trace("< obtain lock");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
if (lock != null) {
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
|
||||
}
|
||||
lockById.delete(lid);
|
||||
txn.commit();
|
||||
}
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (response != null) { // this because the expiration thread can call this method without a real response
|
||||
respond(response, OK);
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void startProject(HttpServletResponse response, String pid, String uid, String lid, byte[] data, String metadata, List<String> transformations) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
if (projectById.contains(pid)) {
|
||||
throw new RuntimeException("Project '" + pid + "' already exists");
|
||||
}
|
||||
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
|
||||
if (lock.type != ALL) {
|
||||
throw new RuntimeException("The lock you have is not enough to start a project");
|
||||
}
|
||||
|
||||
projectById.put(txn, new Project(pid, data, metadata, transformations));
|
||||
txn.commit();
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
|
||||
|
||||
Transaction txn = env.beginTransaction(null, null);
|
||||
|
||||
try {
|
||||
Project project = getProject(pid);
|
||||
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' not found");
|
||||
}
|
||||
|
||||
Lock lock = getLock(lid, pid, uid);
|
||||
|
||||
logger.info("obtained lock: {}", lockToString(lock));
|
||||
|
||||
if (lock.type == ALL) {
|
||||
project.transformations.addAll(transformations);
|
||||
} else {
|
||||
for (String s : transformations) {
|
||||
JSONObject o = new JSONObject(s);
|
||||
|
||||
int type = o.getInt("op_type");
|
||||
String value = o.getString("op_value");
|
||||
if (lock.type == COL) {
|
||||
if (type == COL) {
|
||||
if (value != null && value.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify column '" + value + "'.");
|
||||
}
|
||||
} else if (type == CELL) {
|
||||
String column = value.split(",")[0];
|
||||
if (column != null && column.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify cell '" + value + "' in another column.");
|
||||
}
|
||||
}
|
||||
} else if (lock.type == CELL) {
|
||||
if (type == COL) {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column.");
|
||||
} else if (type == CELL) {
|
||||
if (value != null && value.equals(lock.value)) {
|
||||
project.transformations.add(s);
|
||||
} else {
|
||||
throw new RuntimeException("Can't apply '" + s + "': you have a lock for cell '" + lock.value + "' and you're attempting to modify cell '" + value + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
projectById.put(txn, project);
|
||||
|
||||
txn.commit();
|
||||
} finally {
|
||||
if (txn != null) {
|
||||
txn.abort();
|
||||
txn = null;
|
||||
}
|
||||
}
|
||||
|
||||
respond(response, OK);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void openProject(HttpServletResponse response, String pid) throws Exception {
|
||||
Project project = getProject(pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("status"); writer.value("ok");
|
||||
writer.key("data"); writer.value(project.data);
|
||||
writer.key("metadata"); writer.value(new JSONObject(project.metadata));
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
for (String s : project.transformations) {
|
||||
writer.value(new JSONObject(s));
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
|
||||
|
||||
Project project = getProject(pid);
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
|
||||
writer.object();
|
||||
writer.key("status"); writer.value("ok");
|
||||
writer.key("transformations");
|
||||
writer.array();
|
||||
int size = project.transformations.size();
|
||||
for (int i = rev; i < size; i++) {
|
||||
writer.value(new JSONObject(project.transformations.get(i)));
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
try {
|
||||
writer.key("locks");
|
||||
writer.array();
|
||||
for (Lock lock : cursor) {
|
||||
writer.value(lockToJSON(lock, uid));
|
||||
}
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
|
||||
w.flush();
|
||||
w.close();
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Project getProject(String pid) {
|
||||
Project project = projectById.get(pid);
|
||||
if (project == null) {
|
||||
throw new RuntimeException("Project '" + pid + "' could not be found: are you sure is not managed by another broker?");
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
@Entity
|
||||
static class Project {
|
||||
|
||||
@PrimaryKey
|
||||
String pid;
|
||||
|
||||
List<String> transformations;
|
||||
|
||||
byte[] data;
|
||||
|
||||
String metadata;
|
||||
|
||||
int rev;
|
||||
|
||||
Project(String pid, byte[] data, String metadata, List<String> transformations) {
|
||||
this.pid = pid;
|
||||
this.data = data;
|
||||
this.metadata = metadata;
|
||||
this.transformations = (transformations != null) ? transformations : new ArrayList<String>();
|
||||
this.rev = this.transformations.size();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private Project() {}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
Lock getLock(String lid, String pid, String uid) {
|
||||
Lock lock = lockById.get(lid);
|
||||
checkLock(lock, lid, pid, uid);
|
||||
return lock;
|
||||
}
|
||||
|
||||
void checkLock(Lock lock, String lid, String pid, String uid) {
|
||||
if (lock == null) {
|
||||
throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
|
||||
}
|
||||
|
||||
if (!lock.pid.equals(pid)) {
|
||||
throw new RuntimeException("Lock '" + lock.id + "' is for another project: " + lock.pid);
|
||||
}
|
||||
|
||||
if (!lock.uid.equals(uid)) {
|
||||
throw new RuntimeException("Lock '" + lock.id + "' is owned by another user: " + lock.uid);
|
||||
}
|
||||
}
|
||||
|
||||
Lock getLock(String pid, String uid, int locktype) {
|
||||
Lock lock = null;
|
||||
EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities();
|
||||
|
||||
try {
|
||||
for (Lock l : cursor) {
|
||||
if (uid.equals(l.uid) && (locktype == l.type)) {
|
||||
lock = l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cursor.close();
|
||||
}
|
||||
|
||||
return lock;
|
||||
}
|
||||
|
||||
JSONObject lockToJSON(Lock lock, String uid) throws JSONException {
|
||||
JSONObject o = new JSONObject();
|
||||
if (lock != null) {
|
||||
// NOTE: only the owner of the lock should get the ID,
|
||||
// otherwise others can just fake ownership of other people's locks
|
||||
if (lock.uid.equals(uid)) {
|
||||
o.put("lock", lock.id);
|
||||
}
|
||||
|
||||
o.put("pid", lock.pid);
|
||||
o.put("uid", lock.uid);
|
||||
o.put("type", lock.type);
|
||||
o.put("value", lock.value);
|
||||
o.put("timestamp", lock.timestamp);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
String lockToString(Lock lock) {
|
||||
return lock.id + "," + lock.pid + "," + lock.uid + "," + lock.type + "," + lock.value;
|
||||
}
|
||||
|
||||
@Entity
|
||||
static class Lock {
|
||||
|
||||
@PrimaryKey
|
||||
String id;
|
||||
|
||||
@SecondaryKey(relate=MANY_TO_ONE)
|
||||
String pid;
|
||||
|
||||
String uid;
|
||||
|
||||
int type;
|
||||
|
||||
String value;
|
||||
|
||||
long timestamp;
|
||||
|
||||
Lock(String id, String pid, String uid, int type, String value) {
|
||||
this.id = id;
|
||||
this.pid = pid;
|
||||
this.uid = uid;
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
this.timestamp = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private Lock() {}
|
||||
}
|
||||
}
|
||||
@ -1,452 +0,0 @@
|
||||
package com.google.gridworks.broker.tests;
|
||||
|
||||
import static com.google.gridworks.broker.GridworksBroker.*;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletInputStream;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.AfterSuite;
|
||||
import org.testng.annotations.AfterTest;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeSuite;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.gridworks.broker.GridworksBroker;
|
||||
import com.google.gridworks.broker.GridworksBrokerImpl;
|
||||
|
||||
public class GridworksBrokerTests {
|
||||
|
||||
Logger logger;
|
||||
File data;
|
||||
|
||||
@BeforeSuite public void suite_init() {
|
||||
System.setProperty("log4j.configuration", "tests.log4j.properties");
|
||||
data = new File("data");
|
||||
if (!data.exists()) data.mkdirs();
|
||||
}
|
||||
|
||||
@AfterSuite public void suite_destroy() {
|
||||
for (File f : data.listFiles()) {
|
||||
f.delete();
|
||||
}
|
||||
data.delete();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
ServletConfig config = null;
|
||||
GridworksBroker broker = null;
|
||||
|
||||
@BeforeTest public void test_init() throws Exception {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
config = mock(ServletConfig.class);
|
||||
when(config.getInitParameter("gridworks.data")).thenReturn(data.getAbsolutePath());
|
||||
when(config.getInitParameter("gridworks.development")).thenReturn("true");
|
||||
|
||||
broker = new GridworksBrokerImpl();
|
||||
broker.init(config);
|
||||
}
|
||||
|
||||
@AfterTest public void test_destroy() throws Exception {
|
||||
broker.destroy();
|
||||
broker = null;
|
||||
config = null;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
HttpServletRequest request = null;
|
||||
HttpServletResponse response = null;
|
||||
StringWriter writer = null;
|
||||
|
||||
@BeforeMethod public void setup() throws Exception {
|
||||
request = mock(HttpServletRequest.class);
|
||||
response = mock(HttpServletResponse.class);
|
||||
}
|
||||
|
||||
@AfterMethod public void teardown() throws Exception {
|
||||
response = null;
|
||||
request = null;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
@Test public void testLifeCycle() {
|
||||
Assert.assertTrue(true);
|
||||
}
|
||||
|
||||
@Test public void testService() {
|
||||
try {
|
||||
success(broker, request, response, EXPIRE);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testObtainLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testReleaseLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, RELEASE_LOCK);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testGetStateFailure() {
|
||||
try {
|
||||
failure(broker, request, response, GET_STATE, "pid", "project1934983948", "uid", "testuser", "rev", "0");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenAllLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(ALL), "lockvalue", "1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenColLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(COL), "lockvalue", "1,1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenCellLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(CELL), "lockvalue", "1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLockSimple() {
|
||||
String project = "proj0";
|
||||
String user = "testuser";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- obtain COL lock on project ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
lock = result.getString("lock");
|
||||
|
||||
logger.info("--- release COL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- obtain CELL lock on project ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
lock = result.getString("lock");
|
||||
|
||||
logger.info("--- release CELL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksAllBlocks() {
|
||||
String project = "proj1";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", user);
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- another using asking for any lock will fail ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- same user asking for lower capable locks will return the ALL one ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
String lock2 = result.getString("lock");
|
||||
Assert.assertEquals(lock, lock2);
|
||||
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
lock2 = result.getString("lock");
|
||||
Assert.assertEquals(lock, lock2);
|
||||
|
||||
logger.info("--- release the ALL lock ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksColBlocks() {
|
||||
String project = "proj2";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain COL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- other user must fail to obtain lock on the same COL or ALL ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- but succeed in getting a COL lock on another column or cell ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
String lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- now it's our first user's turn to fail to get lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
|
||||
logger.info("--- release the locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksCellBlocks() {
|
||||
String project = "proj3";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain CELL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- other user must fail to obtain lock on the same CELL, COL or ALL ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- but succeed in getting a CELL lock on a cell in another column ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
String lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- now it's our first user's turn to fail to get lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
|
||||
logger.info("--- release the locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testCompleteProjectLifeCycle() {
|
||||
try {
|
||||
String project = "proj4";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
String data = "blah";
|
||||
String metadata = "{}";
|
||||
String transformations = "[]";
|
||||
String rev = "0";
|
||||
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", user);
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- start project ---");
|
||||
success(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "transformations", transformations);
|
||||
|
||||
logger.info("--- verify project state contains lock ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
|
||||
JSONArray locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 1);
|
||||
JSONObject l = locks.getJSONObject(0);
|
||||
assertJSON(l, "uid", "testuser");
|
||||
Assert.assertEquals(l.getInt("type"), ALL);
|
||||
|
||||
logger.info("--- release ALL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- verify no locks are present ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 0);
|
||||
|
||||
logger.info("--- open project and verify data was loaded correctly ---");
|
||||
result = success(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev);
|
||||
JSONArray result_data = result.getJSONArray("data");
|
||||
Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length);
|
||||
|
||||
JSONArray tt;
|
||||
JSONObject t;
|
||||
|
||||
logger.info("--- obtain column lock ---");
|
||||
String column = "1";
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", column);
|
||||
String col_lock = result.getString("lock");
|
||||
|
||||
logger.info("--- perform column transformation ---");
|
||||
t = new JSONObject();
|
||||
t.put("op_type", COL);
|
||||
t.put("op_value", column); // operate on col 1
|
||||
t.put("value", new JSONObject());
|
||||
tt = new JSONArray();
|
||||
tt.put(t);
|
||||
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
|
||||
|
||||
logger.info("--- make sure transformation was recorded properly ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 1);
|
||||
t = tt.getJSONObject(0);
|
||||
assertJSON(t, "op_value", column);
|
||||
|
||||
logger.info("--- make sure revision numbers in state management work as expected ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 0);
|
||||
|
||||
logger.info("--- perform cell transformation ---");
|
||||
String cell = "1";
|
||||
t = new JSONObject();
|
||||
t.put("op_type", CELL);
|
||||
t.put("op_value", column + "," + cell); // operate on cell at row 1 column 1
|
||||
t.put("value", new JSONObject());
|
||||
tt = new JSONArray();
|
||||
tt.put(t);
|
||||
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
|
||||
|
||||
logger.info("--- make sure transformation was recorded properly ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 2);
|
||||
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 1);
|
||||
t = tt.getJSONObject(0);
|
||||
assertJSON(t, "op_value", column + "," + cell);
|
||||
|
||||
logger.info("--- make sure another user fails to acquire ALL lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
|
||||
logger.info("--- make sure another user fails to acquire COL lock on the same column ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column);
|
||||
|
||||
logger.info("--- make sure another user manages to acquire COL lock on another column ---");
|
||||
String column2 = "2";
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column2);
|
||||
String col_lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- make sure that both locks are present ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 2);
|
||||
|
||||
logger.info("--- make sure we can't escalate our current COL lock to an ALL lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
|
||||
logger.info("--- release column locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", col_lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", col_lock2);
|
||||
|
||||
logger.info("--- make sure the project has no locks ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 0);
|
||||
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
private void assertJSON(JSONObject o, String name, String value) throws JSONException {
|
||||
Assert.assertEquals(o.get(name), value);
|
||||
}
|
||||
|
||||
private JSONObject success(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
return call(true, broker, request, response, service, params);
|
||||
}
|
||||
|
||||
private JSONObject failure(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
return call(false, broker, request, response, service, params);
|
||||
}
|
||||
|
||||
private JSONObject call(boolean successful, GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
if (params != null) {
|
||||
for (int i = 0; i < params.length; ) {
|
||||
String name = params[i++];
|
||||
String value = params[i++];
|
||||
if ("data".equals(name)) {
|
||||
final ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes("UTF-8"));
|
||||
when(request.getInputStream()).thenReturn(new ServletInputStream() {
|
||||
public int read() throws IOException {
|
||||
return inputStream.read();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
when(request.getParameter(name)).thenReturn(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StringWriter writer = new StringWriter();
|
||||
when(response.getWriter()).thenReturn(new PrintWriter(writer));
|
||||
|
||||
broker.process(service, request, response);
|
||||
|
||||
JSONObject result = new JSONObject(writer.toString());
|
||||
|
||||
if (successful) {
|
||||
assertJSON(result, "status", "ok");
|
||||
} else {
|
||||
assertJSON(result, "status", "error");
|
||||
}
|
||||
|
||||
logger.info(result.toString());
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,452 @@
|
||||
package com.google.refine.broker.tests;
|
||||
|
||||
import static com.google.refine.broker.GridworksBroker.*;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletInputStream;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.AfterSuite;
|
||||
import org.testng.annotations.AfterTest;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeSuite;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.broker.GridworksBroker;
|
||||
import com.google.refine.broker.GridworksBrokerImpl;
|
||||
|
||||
public class GridworksBrokerTests {
|
||||
|
||||
Logger logger;
|
||||
File data;
|
||||
|
||||
@BeforeSuite public void suite_init() {
|
||||
System.setProperty("log4j.configuration", "tests.log4j.properties");
|
||||
data = new File("data");
|
||||
if (!data.exists()) data.mkdirs();
|
||||
}
|
||||
|
||||
@AfterSuite public void suite_destroy() {
|
||||
for (File f : data.listFiles()) {
|
||||
f.delete();
|
||||
}
|
||||
data.delete();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
ServletConfig config = null;
|
||||
GridworksBroker broker = null;
|
||||
|
||||
@BeforeTest public void test_init() throws Exception {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
config = mock(ServletConfig.class);
|
||||
when(config.getInitParameter("gridworks.data")).thenReturn(data.getAbsolutePath());
|
||||
when(config.getInitParameter("gridworks.development")).thenReturn("true");
|
||||
|
||||
broker = new GridworksBrokerImpl();
|
||||
broker.init(config);
|
||||
}
|
||||
|
||||
@AfterTest public void test_destroy() throws Exception {
|
||||
broker.destroy();
|
||||
broker = null;
|
||||
config = null;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
HttpServletRequest request = null;
|
||||
HttpServletResponse response = null;
|
||||
StringWriter writer = null;
|
||||
|
||||
@BeforeMethod public void setup() throws Exception {
|
||||
request = mock(HttpServletRequest.class);
|
||||
response = mock(HttpServletResponse.class);
|
||||
}
|
||||
|
||||
@AfterMethod public void teardown() throws Exception {
|
||||
response = null;
|
||||
request = null;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
@Test public void testLifeCycle() {
|
||||
Assert.assertTrue(true);
|
||||
}
|
||||
|
||||
@Test public void testService() {
|
||||
try {
|
||||
success(broker, request, response, EXPIRE);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testObtainLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testReleaseLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, RELEASE_LOCK);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testGetStateFailure() {
|
||||
try {
|
||||
failure(broker, request, response, GET_STATE, "pid", "project1934983948", "uid", "testuser", "rev", "0");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenAllLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(ALL), "lockvalue", "1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenColLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(COL), "lockvalue", "1,1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testBrokenCellLockFailure() {
|
||||
try {
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(CELL), "lockvalue", "1");
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLockSimple() {
|
||||
String project = "proj0";
|
||||
String user = "testuser";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- obtain COL lock on project ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
lock = result.getString("lock");
|
||||
|
||||
logger.info("--- release COL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- obtain CELL lock on project ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
assertJSON(result, "uid", "testuser");
|
||||
lock = result.getString("lock");
|
||||
|
||||
logger.info("--- release CELL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksAllBlocks() {
|
||||
String project = "proj1";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", user);
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- another using asking for any lock will fail ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- same user asking for lower capable locks will return the ALL one ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
String lock2 = result.getString("lock");
|
||||
Assert.assertEquals(lock, lock2);
|
||||
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
lock2 = result.getString("lock");
|
||||
Assert.assertEquals(lock, lock2);
|
||||
|
||||
logger.info("--- release the ALL lock ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksColBlocks() {
|
||||
String project = "proj2";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain COL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- other user must fail to obtain lock on the same COL or ALL ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- but succeed in getting a COL lock on another column or cell ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
String lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- now it's our first user's turn to fail to get lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
|
||||
logger.info("--- release the locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testLocksCellBlocks() {
|
||||
String project = "proj3";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
|
||||
try {
|
||||
logger.info("--- obtain CELL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- other user must fail to obtain lock on the same CELL, COL or ALL ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
|
||||
|
||||
logger.info("--- but succeed in getting a CELL lock on a cell in another column ---");
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
String lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- now it's our first user's turn to fail to get lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
|
||||
|
||||
logger.info("--- release the locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test public void testCompleteProjectLifeCycle() {
|
||||
try {
|
||||
String project = "proj4";
|
||||
String user = "testuser";
|
||||
String user2 = "testuser2";
|
||||
String data = "blah";
|
||||
String metadata = "{}";
|
||||
String transformations = "[]";
|
||||
String rev = "0";
|
||||
|
||||
logger.info("--- obtain ALL lock on project ---");
|
||||
JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
assertJSON(result, "uid", user);
|
||||
String lock = result.getString("lock");
|
||||
|
||||
logger.info("--- start project ---");
|
||||
success(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "transformations", transformations);
|
||||
|
||||
logger.info("--- verify project state contains lock ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
|
||||
JSONArray locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 1);
|
||||
JSONObject l = locks.getJSONObject(0);
|
||||
assertJSON(l, "uid", "testuser");
|
||||
Assert.assertEquals(l.getInt("type"), ALL);
|
||||
|
||||
logger.info("--- release ALL lock on project ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
|
||||
|
||||
logger.info("--- verify no locks are present ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 0);
|
||||
|
||||
logger.info("--- open project and verify data was loaded correctly ---");
|
||||
result = success(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev);
|
||||
JSONArray result_data = result.getJSONArray("data");
|
||||
Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length);
|
||||
|
||||
JSONArray tt;
|
||||
JSONObject t;
|
||||
|
||||
logger.info("--- obtain column lock ---");
|
||||
String column = "1";
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", column);
|
||||
String col_lock = result.getString("lock");
|
||||
|
||||
logger.info("--- perform column transformation ---");
|
||||
t = new JSONObject();
|
||||
t.put("op_type", COL);
|
||||
t.put("op_value", column); // operate on col 1
|
||||
t.put("value", new JSONObject());
|
||||
tt = new JSONArray();
|
||||
tt.put(t);
|
||||
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
|
||||
|
||||
logger.info("--- make sure transformation was recorded properly ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 1);
|
||||
t = tt.getJSONObject(0);
|
||||
assertJSON(t, "op_value", column);
|
||||
|
||||
logger.info("--- make sure revision numbers in state management work as expected ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 0);
|
||||
|
||||
logger.info("--- perform cell transformation ---");
|
||||
String cell = "1";
|
||||
t = new JSONObject();
|
||||
t.put("op_type", CELL);
|
||||
t.put("op_value", column + "," + cell); // operate on cell at row 1 column 1
|
||||
t.put("value", new JSONObject());
|
||||
tt = new JSONArray();
|
||||
tt.put(t);
|
||||
result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
|
||||
|
||||
logger.info("--- make sure transformation was recorded properly ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 2);
|
||||
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
|
||||
tt = result.getJSONArray("transformations");
|
||||
Assert.assertEquals(tt.length(), 1);
|
||||
t = tt.getJSONObject(0);
|
||||
assertJSON(t, "op_value", column + "," + cell);
|
||||
|
||||
logger.info("--- make sure another user fails to acquire ALL lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
|
||||
logger.info("--- make sure another user fails to acquire COL lock on the same column ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column);
|
||||
|
||||
logger.info("--- make sure another user manages to acquire COL lock on another column ---");
|
||||
String column2 = "2";
|
||||
result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column2);
|
||||
String col_lock2 = result.getString("lock");
|
||||
|
||||
logger.info("--- make sure that both locks are present ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 2);
|
||||
|
||||
logger.info("--- make sure we can't escalate our current COL lock to an ALL lock ---");
|
||||
failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
|
||||
|
||||
logger.info("--- release column locks ---");
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", col_lock);
|
||||
success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", col_lock2);
|
||||
|
||||
logger.info("--- make sure the project has no locks ---");
|
||||
result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
|
||||
locks = result.getJSONArray("locks");
|
||||
Assert.assertEquals(locks.length(), 0);
|
||||
|
||||
} catch (Exception e) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
private void assertJSON(JSONObject o, String name, String value) throws JSONException {
|
||||
Assert.assertEquals(o.get(name), value);
|
||||
}
|
||||
|
||||
private JSONObject success(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
return call(true, broker, request, response, service, params);
|
||||
}
|
||||
|
||||
private JSONObject failure(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
return call(false, broker, request, response, service, params);
|
||||
}
|
||||
|
||||
private JSONObject call(boolean successful, GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
|
||||
if (params != null) {
|
||||
for (int i = 0; i < params.length; ) {
|
||||
String name = params[i++];
|
||||
String value = params[i++];
|
||||
if ("data".equals(name)) {
|
||||
final ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes("UTF-8"));
|
||||
when(request.getInputStream()).thenReturn(new ServletInputStream() {
|
||||
public int read() throws IOException {
|
||||
return inputStream.read();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
when(request.getParameter(name)).thenReturn(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StringWriter writer = new StringWriter();
|
||||
when(response.getWriter()).thenReturn(new PrintWriter(writer));
|
||||
|
||||
broker.process(service, request, response);
|
||||
|
||||
JSONObject result = new JSONObject(writer.toString());
|
||||
|
||||
if (successful) {
|
||||
assertJSON(result, "status", "ok");
|
||||
} else {
|
||||
assertJSON(result, "status", "error");
|
||||
}
|
||||
|
||||
logger.info(result.toString());
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -11,13 +11,11 @@
|
||||
<echo message="Building extensions" />
|
||||
<ant dir="sample-extension/" target="build" />
|
||||
<ant dir="jython/" target="build" />
|
||||
<ant dir="rdf-exporter/" target="build" />
|
||||
</target>
|
||||
|
||||
<target name="clean">
|
||||
<echo message="cleaning extensions" />
|
||||
<ant dir="sample-extension/" target="clean" />
|
||||
<ant dir="jython/" target="clean" />
|
||||
<ant dir="rdf-exporter/" target="clean" />
|
||||
</target>
|
||||
</project>
|
||||
@ -1,10 +1,10 @@
|
||||
function init() {
|
||||
// Packages.java.lang.System.err.println("Initializing jython extension");
|
||||
|
||||
Packages.com.google.gridworks.expr.MetaParser.registerLanguageParser(
|
||||
Packages.com.google.refine.expr.MetaParser.registerLanguageParser(
|
||||
"jython",
|
||||
"Jython",
|
||||
Packages.com.google.gridworks.jython.JythonEvaluable.createParser(),
|
||||
Packages.com.google.refine.jython.JythonEvaluable.createParser(),
|
||||
"return value"
|
||||
);
|
||||
}
|
||||
|
||||
@ -1,131 +0,0 @@
|
||||
package com.google.gridworks.jython;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.python.core.Py;
|
||||
import org.python.core.PyException;
|
||||
import org.python.core.PyFunction;
|
||||
import org.python.core.PyNone;
|
||||
import org.python.core.PyObject;
|
||||
import org.python.core.PyString;
|
||||
import org.python.util.PythonInterpreter;
|
||||
|
||||
import com.google.gridworks.expr.EvalError;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.HasFields;
|
||||
import com.google.gridworks.expr.LanguageSpecificParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
|
||||
public class JythonEvaluable implements Evaluable {
|
||||
|
||||
static public LanguageSpecificParser createParser() {
|
||||
return new LanguageSpecificParser() {
|
||||
|
||||
public Evaluable parse(String s) throws ParsingException {
|
||||
return new JythonEvaluable(s);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static final String s_functionName = "___temp___";
|
||||
|
||||
private static PythonInterpreter _engine;
|
||||
|
||||
// FIXME(SM): this initialization logic depends on the fact that the JVM's
|
||||
// current working directory is the root of the Gridworks distributions
|
||||
// or the development checkouts. While this works in practice, it would
|
||||
// be preferable to have a more reliable address space, but since we
|
||||
// don't have access to the servlet context from this class this is
|
||||
// the best we can do for now.
|
||||
static {
|
||||
File libPath = new File("webapp/WEB-INF/lib/jython");
|
||||
if (!libPath.exists() && !libPath.canRead()) {
|
||||
libPath = new File("main/webapp/WEB-INF/lib/jython");
|
||||
if (!libPath.exists() && !libPath.canRead()) {
|
||||
libPath = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (libPath != null) {
|
||||
Properties props = new Properties();
|
||||
props.setProperty("python.path", libPath.getAbsolutePath());
|
||||
PythonInterpreter.initialize(System.getProperties(), props, new String[] { "" });
|
||||
}
|
||||
|
||||
_engine = new PythonInterpreter();
|
||||
}
|
||||
|
||||
public JythonEvaluable(String s) {
|
||||
// indent and create a function out of the code
|
||||
String[] lines = s.split("\r\n|\r|\n");
|
||||
|
||||
StringBuffer sb = new StringBuffer(1024);
|
||||
sb.append("def ");
|
||||
sb.append(s_functionName);
|
||||
sb.append("(value, cell, cells, row, rowIndex):");
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
sb.append("\n ");
|
||||
sb.append(lines[i]);
|
||||
}
|
||||
|
||||
_engine.exec(sb.toString());
|
||||
}
|
||||
|
||||
public Object evaluate(Properties bindings) {
|
||||
try {
|
||||
// call the temporary PyFunction directly
|
||||
Object result = ((PyFunction)_engine.get(s_functionName)).__call__(
|
||||
new PyObject[] {
|
||||
Py.java2py( bindings.get("value") ),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("cell"), bindings),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("cells"), bindings),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("row"), bindings),
|
||||
Py.java2py( bindings.get("rowIndex") )
|
||||
}
|
||||
);
|
||||
|
||||
return unwrap(result);
|
||||
} catch (PyException e) {
|
||||
return new EvalError(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
protected Object unwrap(Object result) {
|
||||
if (result != null) {
|
||||
if (result instanceof JythonObjectWrapper) {
|
||||
return ((JythonObjectWrapper) result)._obj;
|
||||
} else if (result instanceof JythonHasFieldsWrapper) {
|
||||
return ((JythonHasFieldsWrapper) result)._obj;
|
||||
} else if (result instanceof PyString) {
|
||||
return ((PyString) result).asString();
|
||||
} else if (result instanceof PyObject) {
|
||||
return unwrap((PyObject) result);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
protected Object unwrap(PyObject po) {
|
||||
if (po instanceof PyNone) {
|
||||
return null;
|
||||
} else if (po.isNumberType()) {
|
||||
return po.asDouble();
|
||||
} else if (po.isSequenceType()) {
|
||||
Iterator<PyObject> i = po.asIterable().iterator();
|
||||
|
||||
List<Object> list = new ArrayList<Object>();
|
||||
while (i.hasNext()) {
|
||||
list.add(unwrap((Object) i.next()));
|
||||
}
|
||||
|
||||
return list.toArray();
|
||||
} else {
|
||||
return po;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
package com.google.gridworks.jython;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.python.core.Py;
|
||||
import org.python.core.PyObject;
|
||||
|
||||
import com.google.gridworks.expr.HasFields;
|
||||
|
||||
public class JythonHasFieldsWrapper extends PyObject {
|
||||
private static final long serialVersionUID = -1275353513262385099L;
|
||||
|
||||
public HasFields _obj;
|
||||
|
||||
private Properties _bindings;
|
||||
|
||||
public JythonHasFieldsWrapper(HasFields obj, Properties bindings) {
|
||||
_obj = obj;
|
||||
_bindings = bindings;
|
||||
}
|
||||
|
||||
public PyObject __finditem__(PyObject key) {
|
||||
String k = (String) key.__tojava__(String.class);
|
||||
Object v = _obj.getField(k, _bindings);
|
||||
if (v != null) {
|
||||
if (v instanceof PyObject) {
|
||||
return (PyObject) v;
|
||||
} else if (v instanceof HasFields) {
|
||||
return new JythonHasFieldsWrapper((HasFields) v, _bindings);
|
||||
} else if (Py.getAdapter().canAdapt(v)) {
|
||||
return Py.java2py(v);
|
||||
} else {
|
||||
return new JythonObjectWrapper(v);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
package com.google.gridworks.jython;
|
||||
|
||||
import org.python.core.PyObject;
|
||||
|
||||
public class JythonObjectWrapper extends PyObject {
|
||||
private static final long serialVersionUID = -6608115027151667441L;
|
||||
|
||||
public Object _obj;
|
||||
|
||||
public JythonObjectWrapper(Object obj) {
|
||||
_obj = obj;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return _obj.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,131 @@
|
||||
package com.google.refine.jython;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.python.core.Py;
|
||||
import org.python.core.PyException;
|
||||
import org.python.core.PyFunction;
|
||||
import org.python.core.PyNone;
|
||||
import org.python.core.PyObject;
|
||||
import org.python.core.PyString;
|
||||
import org.python.util.PythonInterpreter;
|
||||
|
||||
import com.google.refine.expr.EvalError;
|
||||
import com.google.refine.expr.Evaluable;
|
||||
import com.google.refine.expr.HasFields;
|
||||
import com.google.refine.expr.LanguageSpecificParser;
|
||||
import com.google.refine.expr.ParsingException;
|
||||
|
||||
public class JythonEvaluable implements Evaluable {
|
||||
|
||||
static public LanguageSpecificParser createParser() {
|
||||
return new LanguageSpecificParser() {
|
||||
|
||||
public Evaluable parse(String s) throws ParsingException {
|
||||
return new JythonEvaluable(s);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static final String s_functionName = "___temp___";
|
||||
|
||||
private static PythonInterpreter _engine;
|
||||
|
||||
// FIXME(SM): this initialization logic depends on the fact that the JVM's
|
||||
// current working directory is the root of the Gridworks distributions
|
||||
// or the development checkouts. While this works in practice, it would
|
||||
// be preferable to have a more reliable address space, but since we
|
||||
// don't have access to the servlet context from this class this is
|
||||
// the best we can do for now.
|
||||
static {
|
||||
File libPath = new File("webapp/WEB-INF/lib/jython");
|
||||
if (!libPath.exists() && !libPath.canRead()) {
|
||||
libPath = new File("main/webapp/WEB-INF/lib/jython");
|
||||
if (!libPath.exists() && !libPath.canRead()) {
|
||||
libPath = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (libPath != null) {
|
||||
Properties props = new Properties();
|
||||
props.setProperty("python.path", libPath.getAbsolutePath());
|
||||
PythonInterpreter.initialize(System.getProperties(), props, new String[] { "" });
|
||||
}
|
||||
|
||||
_engine = new PythonInterpreter();
|
||||
}
|
||||
|
||||
public JythonEvaluable(String s) {
|
||||
// indent and create a function out of the code
|
||||
String[] lines = s.split("\r\n|\r|\n");
|
||||
|
||||
StringBuffer sb = new StringBuffer(1024);
|
||||
sb.append("def ");
|
||||
sb.append(s_functionName);
|
||||
sb.append("(value, cell, cells, row, rowIndex):");
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
sb.append("\n ");
|
||||
sb.append(lines[i]);
|
||||
}
|
||||
|
||||
_engine.exec(sb.toString());
|
||||
}
|
||||
|
||||
public Object evaluate(Properties bindings) {
|
||||
try {
|
||||
// call the temporary PyFunction directly
|
||||
Object result = ((PyFunction)_engine.get(s_functionName)).__call__(
|
||||
new PyObject[] {
|
||||
Py.java2py( bindings.get("value") ),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("cell"), bindings),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("cells"), bindings),
|
||||
new JythonHasFieldsWrapper((HasFields) bindings.get("row"), bindings),
|
||||
Py.java2py( bindings.get("rowIndex") )
|
||||
}
|
||||
);
|
||||
|
||||
return unwrap(result);
|
||||
} catch (PyException e) {
|
||||
return new EvalError(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
protected Object unwrap(Object result) {
|
||||
if (result != null) {
|
||||
if (result instanceof JythonObjectWrapper) {
|
||||
return ((JythonObjectWrapper) result)._obj;
|
||||
} else if (result instanceof JythonHasFieldsWrapper) {
|
||||
return ((JythonHasFieldsWrapper) result)._obj;
|
||||
} else if (result instanceof PyString) {
|
||||
return ((PyString) result).asString();
|
||||
} else if (result instanceof PyObject) {
|
||||
return unwrap((PyObject) result);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
protected Object unwrap(PyObject po) {
|
||||
if (po instanceof PyNone) {
|
||||
return null;
|
||||
} else if (po.isNumberType()) {
|
||||
return po.asDouble();
|
||||
} else if (po.isSequenceType()) {
|
||||
Iterator<PyObject> i = po.asIterable().iterator();
|
||||
|
||||
List<Object> list = new ArrayList<Object>();
|
||||
while (i.hasNext()) {
|
||||
list.add(unwrap((Object) i.next()));
|
||||
}
|
||||
|
||||
return list.toArray();
|
||||
} else {
|
||||
return po;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package com.google.refine.jython;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.python.core.Py;
|
||||
import org.python.core.PyObject;
|
||||
|
||||
import com.google.refine.expr.HasFields;
|
||||
|
||||
public class JythonHasFieldsWrapper extends PyObject {
|
||||
private static final long serialVersionUID = -1275353513262385099L;
|
||||
|
||||
public HasFields _obj;
|
||||
|
||||
private Properties _bindings;
|
||||
|
||||
public JythonHasFieldsWrapper(HasFields obj, Properties bindings) {
|
||||
_obj = obj;
|
||||
_bindings = bindings;
|
||||
}
|
||||
|
||||
public PyObject __finditem__(PyObject key) {
|
||||
String k = (String) key.__tojava__(String.class);
|
||||
Object v = _obj.getField(k, _bindings);
|
||||
if (v != null) {
|
||||
if (v instanceof PyObject) {
|
||||
return (PyObject) v;
|
||||
} else if (v instanceof HasFields) {
|
||||
return new JythonHasFieldsWrapper((HasFields) v, _bindings);
|
||||
} else if (Py.getAdapter().canAdapt(v)) {
|
||||
return Py.java2py(v);
|
||||
} else {
|
||||
return new JythonObjectWrapper(v);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package com.google.refine.jython;
|
||||
|
||||
import org.python.core.PyObject;
|
||||
|
||||
public class JythonObjectWrapper extends PyObject {
|
||||
private static final long serialVersionUID = -6608115027151667441L;
|
||||
|
||||
public Object _obj;
|
||||
|
||||
public JythonObjectWrapper(Object obj) {
|
||||
_obj = obj;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return _obj.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
var html = "text/html";
|
||||
var encoding = "UTF-8";
|
||||
var ClientSideResourceManager = Packages.com.google.gridworks.ClientSideResourceManager;
|
||||
var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceManager;
|
||||
|
||||
/*
|
||||
* Function invoked to initialize the extension.
|
||||
@ -39,7 +39,7 @@ function process(path, request, response) {
|
||||
// here's how to pass things into the .vt templates
|
||||
context.someList = ["Superior","Michigan","Huron","Erie","Ontario"];
|
||||
context.someString = "foo";
|
||||
context.someInt = Packages.com.google.gridworks.sampleExtension.SampleUtil.stringArrayLength(context.someList);
|
||||
context.someInt = Packages.com.google.refine.sampleExtension.SampleUtil.stringArrayLength(context.someList);
|
||||
|
||||
send(request, response, "index.vt", context);
|
||||
}
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
package com.google.gridworks.sampleExtension;
|
||||
|
||||
|
||||
public class SampleUtil {
|
||||
static public int stringArrayLength(String[] a) {
|
||||
return a.length;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
package com.google.refine.sampleExtension;
|
||||
|
||||
|
||||
public class SampleUtil {
|
||||
static public int stringArrayLength(String[] a) {
|
||||
return a.length;
|
||||
}
|
||||
}
|
||||
@ -622,7 +622,7 @@ run() {
|
||||
|
||||
CLASSPATH="$GRIDWORKS_CLASSES_DIR${SEP}$GRIDWORKS_LIB_DIR/*"
|
||||
|
||||
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.gridworks.Gridworks"
|
||||
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.refine.Gridworks"
|
||||
|
||||
#echo "$RUN_CMD"
|
||||
#echo ""
|
||||
@ -674,7 +674,7 @@ broker_run() {
|
||||
|
||||
CLASSPATH="$GRIDWORKS_CLASSES_DIR${SEP}$GRIDWORKS_LIB_DIR/*"
|
||||
|
||||
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.gridworks.Gridworks"
|
||||
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.refine.Gridworks"
|
||||
|
||||
#echo "$RUN_CMD"
|
||||
#echo ""
|
||||
|
||||
@ -159,7 +159,7 @@ if ""%ACTION%"" == ""run"" goto doRun
|
||||
|
||||
:doRun
|
||||
set CLASSPATH="%GRIDWORKS_CLASSES_DIR%;%GRIDWORKS_LIB_DIR%\*"
|
||||
"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% -Djava.library.path=%GRIDWORKS_LIB_DIR%/native/windows com.google.gridworks.Gridworks
|
||||
"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% -Djava.library.path=%GRIDWORKS_LIB_DIR%/native/windows com.google.refine.Gridworks
|
||||
goto end
|
||||
|
||||
:doAnt
|
||||
|
||||
@ -1,91 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import edu.mit.simile.butterfly.ButterflyModule;
|
||||
import edu.mit.simile.butterfly.MountPoint;
|
||||
|
||||
|
||||
public class ClientSideResourceManager {
|
||||
final static Logger logger = LoggerFactory.getLogger("gridworks_clientSideResourceManager");
|
||||
|
||||
static public class QualifiedPath {
|
||||
public ButterflyModule module;
|
||||
public String path;
|
||||
public String fullPath;
|
||||
}
|
||||
static public class ClientSideResourceBundle {
|
||||
final protected Set<String> _pathSet = new HashSet<String>();
|
||||
final protected List<QualifiedPath> _pathList = new ArrayList<QualifiedPath>();
|
||||
}
|
||||
|
||||
final static protected Map<String, ClientSideResourceBundle> s_bundles
|
||||
= new HashMap<String, ClientSideResourceBundle>();
|
||||
|
||||
static public void addPaths(
|
||||
String bundleName,
|
||||
ButterflyModule module,
|
||||
String[] paths) {
|
||||
|
||||
ClientSideResourceBundle bundle = s_bundles.get(bundleName);
|
||||
if (bundle == null) {
|
||||
bundle = new ClientSideResourceBundle();
|
||||
s_bundles.put(bundleName, bundle);
|
||||
}
|
||||
|
||||
for (String path : paths) {
|
||||
String fullPath = resolve(module, path);
|
||||
if (fullPath == null) {
|
||||
logger.error("Failed to add paths to unmounted module " + module.getName());
|
||||
break;
|
||||
}
|
||||
if (!bundle._pathSet.contains(fullPath)) {
|
||||
QualifiedPath qualifiedPath = new QualifiedPath();
|
||||
qualifiedPath.module = module;
|
||||
qualifiedPath.path = path;
|
||||
qualifiedPath.fullPath = fullPath;
|
||||
|
||||
bundle._pathSet.add(fullPath);
|
||||
bundle._pathList.add(qualifiedPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static public QualifiedPath[] getPaths(String bundleName) {
|
||||
ClientSideResourceBundle bundle = s_bundles.get(bundleName);
|
||||
if (bundle == null) {
|
||||
return new QualifiedPath[] {};
|
||||
} else {
|
||||
QualifiedPath[] paths = new QualifiedPath[bundle._pathList.size()];
|
||||
bundle._pathList.toArray(paths);
|
||||
return paths;
|
||||
}
|
||||
}
|
||||
|
||||
static protected String resolve(ButterflyModule module, String path) {
|
||||
MountPoint mountPoint = module.getMountPoint();
|
||||
if (mountPoint != null) {
|
||||
String mountPointPath = mountPoint.getMountPoint();
|
||||
if (mountPointPath != null) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
|
||||
boolean slashed = path.startsWith("/");
|
||||
char[] mountPointChars = mountPointPath.toCharArray();
|
||||
|
||||
sb.append(mountPointChars, 0, slashed ? mountPointChars.length - 1 : mountPointChars.length);
|
||||
sb.append(path);
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -1,230 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Timer;
|
||||
import java.util.TimerTask;
|
||||
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.io.FileProjectManager;
|
||||
|
||||
import edu.mit.simile.butterfly.Butterfly;
|
||||
import edu.mit.simile.butterfly.ButterflyModule;
|
||||
|
||||
public class GridworksServlet extends Butterfly {
|
||||
|
||||
static private final String VERSION = "1.5";
|
||||
|
||||
private static final long serialVersionUID = 2386057901503517403L;
|
||||
|
||||
private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir";
|
||||
|
||||
static private GridworksServlet s_singleton;
|
||||
static private File s_dataDir;
|
||||
|
||||
static final private Map<String, Command> commands = new HashMap<String, Command>();
|
||||
|
||||
// timer for periodically saving projects
|
||||
static private Timer _timer;
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("gridworks");
|
||||
|
||||
public static String getVersion() {
|
||||
return VERSION;
|
||||
}
|
||||
|
||||
final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
|
||||
|
||||
static protected class AutoSaveTimerTask extends TimerTask {
|
||||
public void run() {
|
||||
try {
|
||||
ProjectManager.singleton.save(false); // quick, potentially incomplete save
|
||||
} finally {
|
||||
_timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
|
||||
// we don't use scheduleAtFixedRate because that might result in
|
||||
// bunched up events when the computer is put in sleep mode
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected ServletConfig config;
|
||||
|
||||
@Override
|
||||
public void init() throws ServletException {
|
||||
super.init();
|
||||
|
||||
s_singleton = this;
|
||||
|
||||
logger.trace("> initialize");
|
||||
|
||||
String data = getInitParameter("gridworks.data");
|
||||
|
||||
if (data == null) {
|
||||
throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing");
|
||||
}
|
||||
|
||||
s_dataDir = new File(data);
|
||||
FileProjectManager.initialize(s_dataDir);
|
||||
|
||||
if (_timer == null) {
|
||||
_timer = new Timer("autosave");
|
||||
_timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
|
||||
}
|
||||
|
||||
logger.trace("< initialize");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
logger.trace("> destroy");
|
||||
|
||||
// cancel automatic periodic saving and force a complete save.
|
||||
if (_timer != null) {
|
||||
_timer.cancel();
|
||||
_timer = null;
|
||||
}
|
||||
if (ProjectManager.singleton != null) {
|
||||
ProjectManager.singleton.dispose();
|
||||
ProjectManager.singleton = null;
|
||||
}
|
||||
|
||||
this.config = null;
|
||||
|
||||
logger.trace("< destroy");
|
||||
|
||||
super.destroy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
if (request.getPathInfo().startsWith("/command/")) {
|
||||
String commandKey = getCommandKey(request);
|
||||
Command command = commands.get(commandKey);
|
||||
if (command != null) {
|
||||
if (request.getMethod().equals("GET")) {
|
||||
logger.trace("> GET {}", commandKey);
|
||||
command.doGet(request, response);
|
||||
logger.trace("< GET {}", commandKey);
|
||||
} else if (request.getMethod().equals("POST")) {
|
||||
logger.trace("> POST {}", commandKey);
|
||||
command.doPost(request, response);
|
||||
logger.trace("< POST {}", commandKey);
|
||||
} else {
|
||||
response.sendError(405);
|
||||
}
|
||||
} else {
|
||||
response.sendError(404);
|
||||
}
|
||||
} else {
|
||||
super.service(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
protected String getCommandKey(HttpServletRequest request) {
|
||||
// A command path has this format: /command/module-name/command-name/...
|
||||
|
||||
String path = request.getPathInfo().substring("/command/".length());
|
||||
|
||||
int slash1 = path.indexOf('/');
|
||||
if (slash1 >= 0) {
|
||||
int slash2 = path.indexOf('/', slash1 + 1);
|
||||
if (slash2 > 0) {
|
||||
path = path.substring(0, slash2);
|
||||
}
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
private File tempDir = null;
|
||||
|
||||
public File getTempDir() {
|
||||
if (tempDir == null) {
|
||||
File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR);
|
||||
if (tempDir == null) {
|
||||
throw new RuntimeException("This app server doesn't support temp directories");
|
||||
}
|
||||
}
|
||||
return tempDir;
|
||||
}
|
||||
|
||||
public File getTempFile(String name) {
|
||||
return new File(getTempDir(), name);
|
||||
}
|
||||
|
||||
public File getCacheDir(String name) {
|
||||
File dir = new File(new File(s_dataDir, "cache"), name);
|
||||
dir.mkdirs();
|
||||
|
||||
return dir;
|
||||
}
|
||||
|
||||
public String getConfiguration(String name, String def) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a single command.
|
||||
*
|
||||
* @param module the module the command belongs to
|
||||
* @param name command verb for command
|
||||
* @param commandObject object implementing the command
|
||||
* @return true if command was loaded and registered successfully
|
||||
*/
|
||||
protected boolean registerOneCommand(ButterflyModule module, String name, Command commandObject) {
|
||||
return registerOneCommand(module.getName() + "/" + name, commandObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a single command.
|
||||
*
|
||||
* @param path path for command
|
||||
* @param commandObject object implementing the command
|
||||
* @return true if command was loaded and registered successfully
|
||||
*/
|
||||
protected boolean registerOneCommand(String path, Command commandObject) {
|
||||
if (commands.containsKey(path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
commandObject.init(this);
|
||||
commands.put(path, commandObject);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Currently only for test purposes
|
||||
protected boolean unregisterCommand(String verb) {
|
||||
return commands.remove(verb) != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a single command. Used by extensions.
|
||||
*
|
||||
* @param module the module the command belongs to
|
||||
* @param name command verb for command
|
||||
* @param commandObject object implementing the command
|
||||
*
|
||||
* @return true if command was loaded and registered successfully
|
||||
*/
|
||||
static public boolean registerCommand(ButterflyModule module, String commandName, Command commandObject) {
|
||||
return s_singleton.registerOneCommand(module, commandName, commandObject);
|
||||
}
|
||||
|
||||
static public Class<?> getClass(String className) throws ClassNotFoundException {
|
||||
if (className.startsWith("com.metaweb.")) {
|
||||
className = "com.google." + className.substring("com.metaweb.".length());
|
||||
}
|
||||
return Class.forName(className);
|
||||
}
|
||||
}
|
||||
@ -1,128 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.expr.HasFieldsListImpl;
|
||||
import com.google.gridworks.expr.WrappedRow;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class InterProjectModel {
|
||||
static public class ProjectJoin {
|
||||
final public long fromProjectID;
|
||||
final public String fromProjectColumnName;
|
||||
final public long toProjectID;
|
||||
final public String toProjectColumnName;
|
||||
|
||||
final public Map<Object, List<Integer>> valueToRowIndices =
|
||||
new HashMap<Object, List<Integer>>();
|
||||
|
||||
ProjectJoin(
|
||||
long fromProjectID,
|
||||
String fromProjectColumnName,
|
||||
long toProjectID,
|
||||
String toProjectColumnName
|
||||
) {
|
||||
this.fromProjectID = fromProjectID;
|
||||
this.fromProjectColumnName = fromProjectColumnName;
|
||||
this.toProjectID = toProjectID;
|
||||
this.toProjectColumnName = toProjectColumnName;
|
||||
}
|
||||
|
||||
public HasFieldsListImpl getRows(Object value) {
|
||||
if (ExpressionUtils.isNonBlankData(value) && valueToRowIndices.containsKey(value)) {
|
||||
Project toProject = ProjectManager.singleton.getProject(toProjectID);
|
||||
if (toProject != null) {
|
||||
HasFieldsListImpl rows = new HasFieldsListImpl();
|
||||
for (Integer r : valueToRowIndices.get(value)) {
|
||||
Row row = toProject.rows.get(r);
|
||||
rows.add(new WrappedRow(toProject, r, row));
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected Map<String, ProjectJoin> _joins = new HashMap<String, ProjectJoin>();
|
||||
|
||||
public ProjectJoin getJoin(String fromProject, String fromColumn, String toProject, String toColumn) {
|
||||
String key = fromProject + ";" + fromColumn + ";" + toProject + ";" + toColumn;
|
||||
if (!_joins.containsKey(key)) {
|
||||
ProjectJoin join = new ProjectJoin(
|
||||
ProjectManager.singleton.getProjectID(fromProject),
|
||||
fromColumn,
|
||||
ProjectManager.singleton.getProjectID(toProject),
|
||||
toColumn
|
||||
);
|
||||
|
||||
computeJoin(join);
|
||||
|
||||
_joins.put(key, join);
|
||||
}
|
||||
|
||||
return _joins.get(key);
|
||||
}
|
||||
|
||||
public void flushJoinsInvolvingProject(long projectID) {
|
||||
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
|
||||
ProjectJoin join = entry.getValue();
|
||||
if (join.fromProjectID == projectID || join.toProjectID == projectID) {
|
||||
_joins.remove(entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) {
|
||||
for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
|
||||
ProjectJoin join = entry.getValue();
|
||||
if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) ||
|
||||
join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) {
|
||||
_joins.remove(entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void computeJoin(ProjectJoin join) {
|
||||
if (join.fromProjectID < 0 || join.toProjectID < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
Project fromProject = ProjectManager.singleton.getProject(join.fromProjectID);
|
||||
Project toProject = ProjectManager.singleton.getProject(join.toProjectID);
|
||||
if (fromProject == null || toProject == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Column fromColumn = fromProject.columnModel.getColumnByName(join.fromProjectColumnName);
|
||||
Column toColumn = toProject.columnModel.getColumnByName(join.toProjectColumnName);
|
||||
if (fromColumn == null || toColumn == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (Row fromRow : fromProject.rows) {
|
||||
Object value = fromRow.getCellValue(fromColumn.getCellIndex());
|
||||
if (ExpressionUtils.isNonBlankData(value) && !join.valueToRowIndices.containsKey(value)) {
|
||||
join.valueToRowIndices.put(value, new ArrayList<Integer>());
|
||||
}
|
||||
}
|
||||
|
||||
int count = toProject.rows.size();
|
||||
for (int r = 0; r < count; r++) {
|
||||
Row toRow = toProject.rows.get(r);
|
||||
|
||||
Object value = toRow.getCellValue(toColumn.getCellIndex());
|
||||
if (ExpressionUtils.isNonBlankData(value) && join.valueToRowIndices.containsKey(value)) {
|
||||
join.valueToRowIndices.get(value).add(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
/**
|
||||
* Interface for streaming out JSON, either into HTTP responses or
|
||||
* serialization files.
|
||||
*
|
||||
* @author dfhuynh
|
||||
*/
|
||||
public interface Jsonizable {
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException;
|
||||
}
|
||||
@ -1,434 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.tools.tar.TarOutputStream;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.history.HistoryEntryManager;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.preference.PreferenceStore;
|
||||
import com.google.gridworks.preference.TopList;
|
||||
|
||||
/**
|
||||
* ProjectManager is responsible for loading and saving the workspace and projects.
|
||||
*
|
||||
*
|
||||
*/
|
||||
public abstract class ProjectManager {
|
||||
// last n expressions used across all projects
|
||||
static protected final int s_expressionHistoryMax = 100;
|
||||
|
||||
protected Map<Long, ProjectMetadata> _projectsMetadata;
|
||||
protected PreferenceStore _preferenceStore;
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("project_manager");
|
||||
|
||||
/**
|
||||
* What caches the joins between projects.
|
||||
*/
|
||||
transient protected InterProjectModel _interProjectModel = new InterProjectModel();
|
||||
|
||||
/**
|
||||
* Flags
|
||||
*/
|
||||
transient protected int _busy = 0; // heavy operations like creating or importing projects are going on
|
||||
|
||||
/**
|
||||
* While each project's metadata is loaded completely at start-up, each project's raw data
|
||||
* is loaded only when the project is accessed by the user. This is because project
|
||||
* metadata is tiny compared to raw project data. This hash map from project ID to project
|
||||
* is more like a last accessed-last out cache.
|
||||
*/
|
||||
transient protected Map<Long, Project> _projects;
|
||||
|
||||
static public ProjectManager singleton;
|
||||
|
||||
protected ProjectManager(){
|
||||
_projectsMetadata = new HashMap<Long, ProjectMetadata>();
|
||||
_preferenceStore = new PreferenceStore();
|
||||
_projects = new HashMap<Long, Project>();
|
||||
|
||||
preparePreferenceStore(_preferenceStore);
|
||||
}
|
||||
|
||||
public void dispose() {
|
||||
save(true); // complete save
|
||||
|
||||
for (Project project : _projects.values()) {
|
||||
if (project != null) {
|
||||
project.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
_projects.clear();
|
||||
_projectsMetadata.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers the project in the memory of the current session
|
||||
* @param project
|
||||
* @param projectMetadata
|
||||
*/
|
||||
public void registerProject(Project project, ProjectMetadata projectMetadata) {
|
||||
synchronized (this) {
|
||||
_projects.put(project.id, project);
|
||||
_projectsMetadata.put(project.id, projectMetadata);
|
||||
}
|
||||
}
|
||||
//----------Load from data store to memory----------------
|
||||
|
||||
/**
|
||||
* Load project metadata from data storage
|
||||
* @param projectID
|
||||
* @return
|
||||
*/
|
||||
public abstract boolean loadProjectMetadata(long projectID);
|
||||
|
||||
/**
|
||||
* Loads a project from the data store into memory
|
||||
* @param id
|
||||
* @return
|
||||
*/
|
||||
protected abstract Project loadProject(long id);
|
||||
|
||||
//------------Import and Export from Gridworks archive-----------------
|
||||
/**
|
||||
* Import project from a Gridworks archive
|
||||
* @param projectID
|
||||
* @param inputStream
|
||||
* @param gziped
|
||||
* @throws IOException
|
||||
*/
|
||||
public abstract void importProject(long projectID, InputStream inputStream, boolean gziped) throws IOException;
|
||||
|
||||
/**
|
||||
* Export project to a Gridworks archive
|
||||
* @param projectId
|
||||
* @param tos
|
||||
* @throws IOException
|
||||
*/
|
||||
public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException;
|
||||
|
||||
|
||||
//------------Save to record store------------
|
||||
/**
|
||||
* Saves a project and its metadata to the data store
|
||||
* @param id
|
||||
*/
|
||||
public void ensureProjectSaved(long id) {
|
||||
synchronized(this){
|
||||
ProjectMetadata metadata = this.getProjectMetadata(id);
|
||||
if (metadata != null) {
|
||||
try {
|
||||
saveMetadata(metadata, id);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}//FIXME what should be the behaviour if metadata is null? i.e. not found
|
||||
|
||||
Project project = getProject(id);
|
||||
if (project != null && metadata != null && metadata.getModified().after(project.getLastSave())) {
|
||||
try {
|
||||
saveProject(project);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}//FIXME what should be the behaviour if project is null? i.e. not found or loaded.
|
||||
//FIXME what should happen if the metadata is found, but not the project? or vice versa?
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Save project metadata to the data store
|
||||
* @param metadata
|
||||
* @param projectId
|
||||
* @throws Exception
|
||||
*/
|
||||
protected abstract void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception;
|
||||
|
||||
/**
|
||||
* Save project to the data store
|
||||
* @param project
|
||||
*/
|
||||
protected abstract void saveProject(Project project);
|
||||
|
||||
/**
|
||||
* Save workspace and all projects to data store
|
||||
* @param allModified
|
||||
*/
|
||||
public void save(boolean allModified) {
|
||||
if (allModified || _busy == 0) {
|
||||
saveProjects(allModified);
|
||||
saveWorkspace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the workspace to the data store
|
||||
*/
|
||||
protected abstract void saveWorkspace();
|
||||
|
||||
/**
|
||||
* A utility class to prioritize projects for saving, depending on how long ago
|
||||
* they have been changed but have not been saved.
|
||||
*/
|
||||
static protected class SaveRecord {
|
||||
final Project project;
|
||||
final long overdue;
|
||||
|
||||
SaveRecord(Project project, long overdue) {
|
||||
this.project = project;
|
||||
this.overdue = overdue;
|
||||
}
|
||||
}
|
||||
|
||||
static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
|
||||
static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
|
||||
|
||||
/**
|
||||
* Saves all projects to the data store
|
||||
* @param allModified
|
||||
*/
|
||||
protected void saveProjects(boolean allModified) {
|
||||
List<SaveRecord> records = new ArrayList<SaveRecord>();
|
||||
Date startTimeOfSave = new Date();
|
||||
|
||||
synchronized (this) {
|
||||
for (long id : _projectsMetadata.keySet()) {
|
||||
ProjectMetadata metadata = getProjectMetadata(id);
|
||||
Project project = _projects.get(id); // don't call getProject() as that will load the project.
|
||||
|
||||
if (project != null) {
|
||||
boolean hasUnsavedChanges =
|
||||
metadata.getModified().getTime() > project.getLastSave().getTime();
|
||||
|
||||
if (hasUnsavedChanges) {
|
||||
long msecsOverdue = startTimeOfSave.getTime() - project.getLastSave().getTime();
|
||||
|
||||
records.add(new SaveRecord(project, msecsOverdue));
|
||||
|
||||
} else if (startTimeOfSave.getTime() - project.getLastSave().getTime() > s_projectFlushDelay) {
|
||||
/*
|
||||
* It's been a while since the project was last saved and it hasn't been
|
||||
* modified. We can safely remove it from the cache to save some memory.
|
||||
*/
|
||||
_projects.remove(id).dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (records.size() > 0) {
|
||||
Collections.sort(records, new Comparator<SaveRecord>() {
|
||||
public int compare(SaveRecord o1, SaveRecord o2) {
|
||||
if (o1.overdue < o2.overdue) {
|
||||
return 1;
|
||||
} else if (o1.overdue > o2.overdue) {
|
||||
return -1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(allModified ?
|
||||
"Saving all modified projects ..." :
|
||||
"Saving some modified projects ..."
|
||||
);
|
||||
|
||||
for (int i = 0;
|
||||
i < records.size() &&
|
||||
(allModified || (new Date().getTime() - startTimeOfSave.getTime() < s_quickSaveTimeout));
|
||||
i++) {
|
||||
|
||||
try {
|
||||
saveProject(records.get(i).project);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//--------------Get from memory--------------
|
||||
/**
|
||||
* Gets the InterProjectModel from memory
|
||||
*/
|
||||
public InterProjectModel getInterProjectModel() {
|
||||
return _interProjectModel;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Gets the project metadata from memory
|
||||
* Requires that the metadata has already been loaded from the data store
|
||||
* @param id
|
||||
* @return
|
||||
*/
|
||||
public ProjectMetadata getProjectMetadata(long id) {
|
||||
return _projectsMetadata.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the project metadata from memory
|
||||
* Requires that the metadata has already been loaded from the data store
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
public ProjectMetadata getProjectMetadata(String name) {
|
||||
for (ProjectMetadata pm : _projectsMetadata.values()) {
|
||||
if (pm.getName().equals(name)) {
|
||||
return pm;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to find the project id when given a project name
|
||||
* Requires that all project metadata exists has been loaded to memory from the data store
|
||||
* @param name
|
||||
* The name of the project
|
||||
* @return
|
||||
* The id of the project, or -1 if it cannot be found
|
||||
*/
|
||||
public long getProjectID(String name) {
|
||||
for (Entry<Long, ProjectMetadata> entry : _projectsMetadata.entrySet()) {
|
||||
if (entry.getValue().getName().equals(name)) {
|
||||
return entry.getKey();
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Gets all the project Metadata currently held in memory
|
||||
* @return
|
||||
*/
|
||||
public Map<Long, ProjectMetadata> getAllProjectMetadata() {
|
||||
return _projectsMetadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the required project from the data store
|
||||
* If project does not already exist in memory, it is loaded from the data store
|
||||
* @param id
|
||||
* the id of the project
|
||||
* @return
|
||||
* the project with the matching id, or null if it can't be found
|
||||
*/
|
||||
public Project getProject(long id) {
|
||||
synchronized (this) {
|
||||
if (_projects.containsKey(id)) {
|
||||
return _projects.get(id);
|
||||
} else {
|
||||
Project project = loadProject(id);
|
||||
|
||||
_projects.put(id, project);
|
||||
|
||||
return project;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the preference store
|
||||
* @return
|
||||
*/
|
||||
public PreferenceStore getPreferenceStore() {
|
||||
return _preferenceStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all expressions from the preference store
|
||||
* @return
|
||||
*/
|
||||
public List<String> getExpressions() {
|
||||
return ((TopList) _preferenceStore.get("scripting.expressions")).getList();
|
||||
}
|
||||
|
||||
/**
|
||||
* The history entry manager deals with changes
|
||||
* @return manager for handling history
|
||||
*/
|
||||
public abstract HistoryEntryManager getHistoryEntryManager();
|
||||
|
||||
//-------------remove project-----------
|
||||
|
||||
/**
|
||||
* Remove the project from the data store
|
||||
* @param project
|
||||
*/
|
||||
public void deleteProject(Project project) {
|
||||
deleteProject(project.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove project from data store
|
||||
* @param projectID
|
||||
*/
|
||||
public abstract void deleteProject(long projectID);
|
||||
|
||||
/**
|
||||
* Removes project from memory
|
||||
* @param projectID
|
||||
*/
|
||||
protected void removeProject(long projectID){
|
||||
if (_projects.containsKey(projectID)) {
|
||||
_projects.remove(projectID).dispose();
|
||||
}
|
||||
if (_projectsMetadata.containsKey(projectID)) {
|
||||
_projectsMetadata.remove(projectID);
|
||||
}
|
||||
}
|
||||
|
||||
//--------------Miscellaneous-----------
|
||||
|
||||
/**
|
||||
* Sets the flag for long running operations
|
||||
* @param busy
|
||||
*/
|
||||
public void setBusy(boolean busy) {
|
||||
synchronized (this) {
|
||||
if (busy) {
|
||||
_busy++;
|
||||
} else {
|
||||
_busy--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Add the latest expression to the preference store
|
||||
* @param s
|
||||
*/
|
||||
public void addLatestExpression(String s) {
|
||||
synchronized (this) {
|
||||
((TopList) _preferenceStore.get("scripting.expressions")).add(s);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @param ps
|
||||
*/
|
||||
static protected void preparePreferenceStore(PreferenceStore ps) {
|
||||
ps.put("scripting.expressions", new TopList(s_expressionHistoryMax));
|
||||
}
|
||||
}
|
||||
@ -1,198 +0,0 @@
|
||||
package com.google.gridworks;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.preference.PreferenceStore;
|
||||
import com.google.gridworks.preference.TopList;
|
||||
import com.google.gridworks.util.JSONUtilities;
|
||||
import com.google.gridworks.util.ParsingUtilities;
|
||||
|
||||
public class ProjectMetadata implements Jsonizable {
|
||||
private final Date _created;
|
||||
private Date _modified;
|
||||
private String _name;
|
||||
private String _password;
|
||||
|
||||
private String _encoding;
|
||||
private int _encodingConfidence;
|
||||
|
||||
private Map<String, Serializable> _customMetadata = new HashMap<String, Serializable>();
|
||||
private PreferenceStore _preferenceStore = new PreferenceStore();
|
||||
|
||||
final Logger logger = LoggerFactory.getLogger("project_metadata");
|
||||
|
||||
protected ProjectMetadata(Date date) {
|
||||
_created = date;
|
||||
preparePreferenceStore(_preferenceStore);
|
||||
}
|
||||
|
||||
public ProjectMetadata() {
|
||||
_created = new Date();
|
||||
_modified = _created;
|
||||
preparePreferenceStore(_preferenceStore);
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("name"); writer.value(_name);
|
||||
writer.key("created"); writer.value(ParsingUtilities.dateToString(_created));
|
||||
writer.key("modified"); writer.value(ParsingUtilities.dateToString(_modified));
|
||||
|
||||
if ("save".equals(options.getProperty("mode"))) {
|
||||
writer.key("password"); writer.value(_password);
|
||||
|
||||
writer.key("encoding"); writer.value(_encoding);
|
||||
writer.key("encodingConfidence"); writer.value(_encodingConfidence);
|
||||
|
||||
writer.key("customMetadata"); writer.object();
|
||||
for (String key : _customMetadata.keySet()) {
|
||||
Serializable value = _customMetadata.get(key);
|
||||
writer.key(key);
|
||||
writer.value(value);
|
||||
}
|
||||
writer.endObject();
|
||||
|
||||
writer.key("preferences"); _preferenceStore.write(writer, options);
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void write(JSONWriter jsonWriter) throws Exception {
|
||||
Properties options = new Properties();
|
||||
options.setProperty("mode", "save");
|
||||
|
||||
write(jsonWriter, options);
|
||||
}
|
||||
|
||||
static public ProjectMetadata loadFromJSON(JSONObject obj) {
|
||||
ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date()));
|
||||
|
||||
pm._modified = JSONUtilities.getDate(obj, "modified", new Date());
|
||||
pm._name = JSONUtilities.getString(obj, "name", "<Error recovering project name>");
|
||||
pm._password = JSONUtilities.getString(obj, "password", "");
|
||||
|
||||
pm._encoding = JSONUtilities.getString(obj, "encoding", "");
|
||||
pm._encodingConfidence = JSONUtilities.getInt(obj, "encodingConfidence", 0);
|
||||
|
||||
if (obj.has("preferences") && !obj.isNull("preferences")) {
|
||||
try {
|
||||
pm._preferenceStore.load(obj.getJSONObject("preferences"));
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.has("expressions") && !obj.isNull("expressions")) { // backward compatibility
|
||||
try {
|
||||
((TopList) pm._preferenceStore.get("scripting.expressions"))
|
||||
.load(obj.getJSONArray("expressions"));
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.has("customMetadata") && !obj.isNull("customMetadata")) {
|
||||
try {
|
||||
JSONObject obj2 = obj.getJSONObject("customMetadata");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> keys = obj2.keys();
|
||||
while (keys.hasNext()) {
|
||||
String key = keys.next();
|
||||
Object value = obj2.get(key);
|
||||
if (value != null && value instanceof Serializable) {
|
||||
pm._customMetadata.put(key, (Serializable) value);
|
||||
}
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
return pm;
|
||||
}
|
||||
|
||||
static protected void preparePreferenceStore(PreferenceStore ps) {
|
||||
ProjectManager.preparePreferenceStore(ps);
|
||||
// Any project specific preferences?
|
||||
}
|
||||
|
||||
public Date getCreated() {
|
||||
return _created;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this._name = name;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return _name;
|
||||
}
|
||||
|
||||
public void setEncoding(String encoding) {
|
||||
this._encoding = encoding;
|
||||
}
|
||||
|
||||
public String getEncoding() {
|
||||
return _encoding;
|
||||
}
|
||||
|
||||
public void setEncodingConfidence(int confidence) {
|
||||
this._encodingConfidence = confidence;
|
||||
}
|
||||
|
||||
public void setEncodingConfidence(String confidence) {
|
||||
if (confidence != null) {
|
||||
this.setEncodingConfidence(Integer.parseInt(confidence));
|
||||
}
|
||||
}
|
||||
|
||||
public int getEncodingConfidence() {
|
||||
return _encodingConfidence;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this._password = password;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return _password;
|
||||
}
|
||||
|
||||
public Date getModified() {
|
||||
return _modified;
|
||||
}
|
||||
|
||||
public void updateModified() {
|
||||
_modified = new Date();
|
||||
}
|
||||
|
||||
public PreferenceStore getPreferenceStore() {
|
||||
return _preferenceStore;
|
||||
}
|
||||
|
||||
public Serializable getCustomMetadata(String key) {
|
||||
return _customMetadata.get(key);
|
||||
}
|
||||
|
||||
public void setCustomMetadata(String key, Serializable value) {
|
||||
if (value == null) {
|
||||
_customMetadata.remove(key);
|
||||
} else {
|
||||
_customMetadata.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.Jsonizable;
|
||||
|
||||
/**
|
||||
* Store a value and its text label, in case the value is not a string itself.
|
||||
* For instance, if a value is a date, then its label can be one particular
|
||||
* rendering of that date.
|
||||
*
|
||||
* Facet choices that are presented to the user as text are stored as decorated values.
|
||||
*/
|
||||
public class DecoratedValue implements Jsonizable {
|
||||
final public Object value;
|
||||
final public String label;
|
||||
|
||||
public DecoratedValue(Object value, String label) {
|
||||
this.value = value;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
writer.object();
|
||||
writer.key("v"); writer.value(value);
|
||||
writer.key("l"); writer.value(label);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,214 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.Jsonizable;
|
||||
import com.google.gridworks.browsing.facets.Facet;
|
||||
import com.google.gridworks.browsing.facets.ListFacet;
|
||||
import com.google.gridworks.browsing.facets.RangeFacet;
|
||||
import com.google.gridworks.browsing.facets.ScatterplotFacet;
|
||||
import com.google.gridworks.browsing.facets.TextSearchFacet;
|
||||
import com.google.gridworks.browsing.facets.TimeRangeFacet;
|
||||
import com.google.gridworks.browsing.util.ConjunctiveFilteredRecords;
|
||||
import com.google.gridworks.browsing.util.ConjunctiveFilteredRows;
|
||||
import com.google.gridworks.browsing.util.FilteredRecordsAsFilteredRows;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Faceted browsing engine.
|
||||
*/
|
||||
public class Engine implements Jsonizable {
|
||||
static public enum Mode {
|
||||
RowBased,
|
||||
RecordBased
|
||||
}
|
||||
|
||||
public final static String INCLUDE_DEPENDENT = "includeDependent";
|
||||
public final static String MODE = "mode";
|
||||
public final static String MODE_ROW_BASED = "row-based";
|
||||
public final static String MODE_RECORD_BASED = "record-based";
|
||||
|
||||
protected Project _project;
|
||||
protected List<Facet> _facets = new LinkedList<Facet>();
|
||||
protected Mode _mode = Mode.RowBased;
|
||||
|
||||
static public String modeToString(Mode mode) {
|
||||
return mode == Mode.RowBased ? MODE_ROW_BASED : MODE_RECORD_BASED;
|
||||
}
|
||||
static public Mode stringToMode(String s) {
|
||||
return MODE_ROW_BASED.equals(s) ? Mode.RowBased : Mode.RecordBased;
|
||||
}
|
||||
|
||||
public Engine(Project project) {
|
||||
_project = project;
|
||||
}
|
||||
|
||||
public Mode getMode() {
|
||||
return _mode;
|
||||
}
|
||||
public void setMode(Mode mode) {
|
||||
_mode = mode;
|
||||
}
|
||||
|
||||
public FilteredRows getAllRows() {
|
||||
return new FilteredRows() {
|
||||
@Override
|
||||
public void accept(Project project, RowVisitor visitor) {
|
||||
try {
|
||||
visitor.start(project);
|
||||
|
||||
int c = project.rows.size();
|
||||
for (int rowIndex = 0; rowIndex < c; rowIndex++) {
|
||||
Row row = project.rows.get(rowIndex);
|
||||
visitor.visit(project, rowIndex, row);
|
||||
}
|
||||
} finally {
|
||||
visitor.end(project);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public FilteredRows getAllFilteredRows() {
|
||||
return getFilteredRows(null);
|
||||
}
|
||||
|
||||
public FilteredRows getFilteredRows(Facet except) {
|
||||
if (_mode == Mode.RecordBased) {
|
||||
return new FilteredRecordsAsFilteredRows(getFilteredRecords(except));
|
||||
} else if (_mode == Mode.RowBased) {
|
||||
ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows();
|
||||
for (Facet facet : _facets) {
|
||||
if (facet != except) {
|
||||
RowFilter rowFilter = facet.getRowFilter(_project);
|
||||
if (rowFilter != null) {
|
||||
cfr.add(rowFilter);
|
||||
}
|
||||
}
|
||||
}
|
||||
return cfr;
|
||||
}
|
||||
throw new InternalError("Unknown mode.");
|
||||
}
|
||||
|
||||
public FilteredRecords getAllRecords() {
|
||||
return new FilteredRecords() {
|
||||
@Override
|
||||
public void accept(Project project, RecordVisitor visitor) {
|
||||
try {
|
||||
visitor.start(project);
|
||||
|
||||
int c = project.recordModel.getRecordCount();
|
||||
for (int r = 0; r < c; r++) {
|
||||
visitor.visit(project, project.recordModel.getRecord(r));
|
||||
}
|
||||
} finally {
|
||||
visitor.end(project);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public FilteredRecords getFilteredRecords() {
|
||||
return getFilteredRecords(null);
|
||||
}
|
||||
|
||||
public FilteredRecords getFilteredRecords(Facet except) {
|
||||
if (_mode == Mode.RecordBased) {
|
||||
ConjunctiveFilteredRecords cfr = new ConjunctiveFilteredRecords();
|
||||
for (Facet facet : _facets) {
|
||||
if (facet != except) {
|
||||
RecordFilter recordFilter = facet.getRecordFilter(_project);
|
||||
if (recordFilter != null) {
|
||||
cfr.add(recordFilter);
|
||||
}
|
||||
}
|
||||
}
|
||||
return cfr;
|
||||
}
|
||||
throw new InternalError("This method should not be called when the engine is not in record mode.");
|
||||
}
|
||||
|
||||
public void initializeFromJSON(JSONObject o) throws Exception {
|
||||
if (o == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (o.has("facets") && !o.isNull("facets")) {
|
||||
JSONArray a = o.getJSONArray("facets");
|
||||
int length = a.length();
|
||||
|
||||
for (int i = 0; i < length; i++) {
|
||||
JSONObject fo = a.getJSONObject(i);
|
||||
String type = fo.has("type") ? fo.getString("type") : "list";
|
||||
|
||||
Facet facet = null;
|
||||
if ("list".equals(type)) {
|
||||
facet = new ListFacet();
|
||||
} else if ("range".equals(type)) {
|
||||
facet = new RangeFacet();
|
||||
} else if ("timerange".equals(type)) {
|
||||
facet = new TimeRangeFacet();
|
||||
} else if ("scatterplot".equals(type)) {
|
||||
facet = new ScatterplotFacet();
|
||||
} else if ("text".equals(type)) {
|
||||
facet = new TextSearchFacet();
|
||||
}
|
||||
|
||||
if (facet != null) {
|
||||
facet.initializeFromJSON(_project, fo);
|
||||
_facets.add(facet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// for backward compatibility
|
||||
if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) {
|
||||
_mode = o.getBoolean(INCLUDE_DEPENDENT) ? Mode.RecordBased : Mode.RowBased;
|
||||
}
|
||||
|
||||
if (o.has(MODE) && !o.isNull(MODE)) {
|
||||
_mode = MODE_ROW_BASED.equals(o.getString(MODE)) ? Mode.RowBased : Mode.RecordBased;
|
||||
}
|
||||
}
|
||||
|
||||
public void computeFacets() throws JSONException {
|
||||
if (_mode == Mode.RowBased) {
|
||||
for (Facet facet : _facets) {
|
||||
FilteredRows filteredRows = getFilteredRows(facet);
|
||||
|
||||
facet.computeChoices(_project, filteredRows);
|
||||
}
|
||||
} else if (_mode == Mode.RecordBased) {
|
||||
for (Facet facet : _facets) {
|
||||
FilteredRecords filteredRecords = getFilteredRecords(facet);
|
||||
|
||||
facet.computeChoices(_project, filteredRecords);
|
||||
}
|
||||
} else {
|
||||
throw new InternalError("Unknown mode.");
|
||||
}
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("facets");
|
||||
writer.array();
|
||||
for (Facet facet : _facets) {
|
||||
facet.write(writer, options);
|
||||
}
|
||||
writer.endArray();
|
||||
writer.key(MODE); writer.value(_mode == Mode.RowBased ? MODE_ROW_BASED : MODE_RECORD_BASED);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
/**
|
||||
* Interface for anything that can decide which records match and which don't
|
||||
* based on some particular criteria.
|
||||
*/
|
||||
public interface FilteredRecords {
|
||||
/**
|
||||
* Go through the records of the given project, determine which match and which don't,
|
||||
* and call visitor.visit() on those that match
|
||||
*
|
||||
* @param project
|
||||
* @param visitor
|
||||
*/
|
||||
public void accept(Project project, RecordVisitor visitor);
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
/**
|
||||
* Interface for anything that can decide which rows match and which rows don't match
|
||||
* based on some particular criteria.
|
||||
*/
|
||||
public interface FilteredRows {
|
||||
/**
|
||||
* Go through the rows of the given project, determine which match and which don't,
|
||||
* and call visitor.visit() on those that match, and possibly their context and
|
||||
* dependent rows.
|
||||
*
|
||||
* @param project
|
||||
* @param visitor
|
||||
*/
|
||||
public void accept(Project project, RowVisitor visitor);
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
/**
|
||||
* Interface for judging if a particular record matches or doesn't match some
|
||||
* particular criterion, such as a facet constraint.
|
||||
*/
|
||||
public interface RecordFilter {
|
||||
public boolean filterRecord(Project project, Record record);
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
/**
|
||||
* Interface for visiting records one by one. The records visited are only those that match some
|
||||
* particular criteria, such as facets' constraints.
|
||||
*/
|
||||
public interface RecordVisitor {
|
||||
public void start(Project project); // called before any visit() call
|
||||
|
||||
public boolean visit(
|
||||
Project project,
|
||||
Record record
|
||||
);
|
||||
|
||||
public void end(Project project); // called after all visit() calls
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Interface for judging if a particular row matches or doesn't match some
|
||||
* particular criterion, such as a facet constraint.
|
||||
*/
|
||||
public interface RowFilter {
|
||||
public boolean filterRow(Project project, int rowIndex, Row row);
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
package com.google.gridworks.browsing;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Interface for visiting rows one by one. The rows visited are only those that match some
|
||||
* particular criteria, such as facets' constraints.
|
||||
*/
|
||||
public interface RowVisitor {
|
||||
public void start(Project project); // called before any visit() call
|
||||
|
||||
public boolean visit(
|
||||
Project project,
|
||||
int rowIndex, // zero-based row index
|
||||
Row row
|
||||
);
|
||||
|
||||
public void end(Project project); // called after all visit() calls
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.Jsonizable;
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
/**
|
||||
* Interface of facets.
|
||||
*/
|
||||
public interface Facet extends Jsonizable {
|
||||
public RowFilter getRowFilter(Project project);
|
||||
|
||||
public RecordFilter getRecordFilter(Project project);
|
||||
|
||||
public void computeChoices(Project project, FilteredRows filteredRows);
|
||||
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords);
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception;
|
||||
}
|
||||
@ -1,255 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.browsing.DecoratedValue;
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.filters.AllRowsRecordFilter;
|
||||
import com.google.gridworks.browsing.filters.AnyRowRecordFilter;
|
||||
import com.google.gridworks.browsing.filters.ExpressionEqualRowFilter;
|
||||
import com.google.gridworks.browsing.util.ExpressionNominalValueGrouper;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.MetaParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.util.JSONUtilities;
|
||||
|
||||
public class ListFacet implements Facet {
|
||||
/*
|
||||
* Configuration
|
||||
*/
|
||||
protected String _name;
|
||||
protected String _expression;
|
||||
protected String _columnName;
|
||||
protected boolean _invert;
|
||||
|
||||
// If true, then facet won't show the blank and error choices
|
||||
protected boolean _omitBlank;
|
||||
protected boolean _omitError;
|
||||
|
||||
protected List<NominalFacetChoice> _selection = new LinkedList<NominalFacetChoice>();
|
||||
protected boolean _selectBlank;
|
||||
protected boolean _selectError;
|
||||
|
||||
/*
|
||||
* Derived configuration
|
||||
*/
|
||||
protected int _cellIndex;
|
||||
protected Evaluable _eval;
|
||||
protected String _errorMessage;
|
||||
|
||||
/*
|
||||
* Computed results
|
||||
*/
|
||||
protected List<NominalFacetChoice> _choices = new LinkedList<NominalFacetChoice>();
|
||||
protected int _blankCount;
|
||||
protected int _errorCount;
|
||||
|
||||
public ListFacet() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("name"); writer.value(_name);
|
||||
writer.key("expression"); writer.value(_expression);
|
||||
writer.key("columnName"); writer.value(_columnName);
|
||||
writer.key("invert"); writer.value(_invert);
|
||||
|
||||
if (_errorMessage != null) {
|
||||
writer.key("error"); writer.value(_errorMessage);
|
||||
} else if (_choices.size() > getLimit()) {
|
||||
writer.key("error"); writer.value("Too many choices");
|
||||
} else {
|
||||
writer.key("choices"); writer.array();
|
||||
for (NominalFacetChoice choice : _choices) {
|
||||
choice.write(writer, options);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
if (!_omitBlank && (_selectBlank || _blankCount > 0)) {
|
||||
writer.key("blankChoice");
|
||||
writer.object();
|
||||
writer.key("s"); writer.value(_selectBlank);
|
||||
writer.key("c"); writer.value(_blankCount);
|
||||
writer.endObject();
|
||||
}
|
||||
if (!_omitError && (_selectError || _errorCount > 0)) {
|
||||
writer.key("errorChoice");
|
||||
writer.object();
|
||||
writer.key("s"); writer.value(_selectError);
|
||||
writer.key("c"); writer.value(_errorCount);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
protected int getLimit() {
|
||||
Object v = ProjectManager.singleton.getPreferenceStore().get("ui.browsing.listFacet.limit");
|
||||
if (v != null) {
|
||||
if (v instanceof Number) {
|
||||
return ((Number) v).intValue();
|
||||
} else {
|
||||
try {
|
||||
int n = Integer.parseInt(v.toString());
|
||||
return n;
|
||||
} catch (NumberFormatException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
return 2000;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
_name = o.getString("name");
|
||||
_expression = o.getString("expression");
|
||||
_columnName = o.getString("columnName");
|
||||
_invert = o.has("invert") && o.getBoolean("invert");
|
||||
|
||||
if (_columnName.length() > 0) {
|
||||
Column column = project.columnModel.getColumnByName(_columnName);
|
||||
if (column != null) {
|
||||
_cellIndex = column.getCellIndex();
|
||||
} else {
|
||||
_errorMessage = "No column named " + _columnName;
|
||||
}
|
||||
} else {
|
||||
_cellIndex = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
_eval = MetaParser.parse(_expression);
|
||||
} catch (ParsingException e) {
|
||||
_errorMessage = e.getMessage();
|
||||
}
|
||||
|
||||
_selection.clear();
|
||||
|
||||
JSONArray a = o.getJSONArray("selection");
|
||||
int length = a.length();
|
||||
|
||||
for (int i = 0; i < length; i++) {
|
||||
JSONObject oc = a.getJSONObject(i);
|
||||
JSONObject ocv = oc.getJSONObject("v");
|
||||
|
||||
DecoratedValue decoratedValue = new DecoratedValue(
|
||||
ocv.get("v"), ocv.getString("l"));
|
||||
|
||||
NominalFacetChoice nominalFacetChoice = new NominalFacetChoice(decoratedValue);
|
||||
nominalFacetChoice.selected = true;
|
||||
|
||||
_selection.add(nominalFacetChoice);
|
||||
}
|
||||
|
||||
_omitBlank = JSONUtilities.getBoolean(o, "omitBlank", false);
|
||||
_omitError = JSONUtilities.getBoolean(o, "omitError", false);
|
||||
|
||||
_selectBlank = JSONUtilities.getBoolean(o, "selectBlank", false);
|
||||
_selectError = JSONUtilities.getBoolean(o, "selectError", false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RowFilter getRowFilter(Project project) {
|
||||
return
|
||||
_eval == null ||
|
||||
_errorMessage != null ||
|
||||
(_selection.size() == 0 && !_selectBlank && !_selectError) ?
|
||||
null :
|
||||
new ExpressionEqualRowFilter(
|
||||
_eval,
|
||||
_columnName,
|
||||
_cellIndex,
|
||||
createMatches(),
|
||||
_selectBlank,
|
||||
_selectError,
|
||||
_invert);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordFilter getRecordFilter(Project project) {
|
||||
RowFilter rowFilter = getRowFilter(project);
|
||||
return rowFilter == null ? null :
|
||||
(_invert ?
|
||||
new AllRowsRecordFilter(rowFilter) :
|
||||
new AnyRowRecordFilter(rowFilter));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void computeChoices(Project project, FilteredRows filteredRows) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
ExpressionNominalValueGrouper grouper =
|
||||
new ExpressionNominalValueGrouper(_eval, _columnName, _cellIndex);
|
||||
|
||||
filteredRows.accept(project, grouper);
|
||||
|
||||
postProcessGrouper(grouper);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
ExpressionNominalValueGrouper grouper =
|
||||
new ExpressionNominalValueGrouper(_eval, _columnName, _cellIndex);
|
||||
|
||||
filteredRecords.accept(project, grouper);
|
||||
|
||||
postProcessGrouper(grouper);
|
||||
}
|
||||
}
|
||||
|
||||
protected void postProcessGrouper(ExpressionNominalValueGrouper grouper) {
|
||||
_choices.clear();
|
||||
_choices.addAll(grouper.choices.values());
|
||||
|
||||
for (NominalFacetChoice choice : _selection) {
|
||||
String valueString = choice.decoratedValue.value.toString();
|
||||
|
||||
if (grouper.choices.containsKey(valueString)) {
|
||||
grouper.choices.get(valueString).selected = true;
|
||||
} else {
|
||||
/*
|
||||
* A selected choice can have zero count if it is selected together
|
||||
* with other choices, and some other facets' constraints eliminate
|
||||
* all rows projected to this choice altogether. For example, if you
|
||||
* select both "car" and "bicycle" in the "type of vehicle" facet, and
|
||||
* then constrain the "wheels" facet to more than 2, then the "bicycle"
|
||||
* choice now has zero count even if it's still selected. The grouper
|
||||
* won't be able to detect the "bicycle" choice, so we need to inject
|
||||
* that choice into the choice list ourselves.
|
||||
*/
|
||||
choice.count = 0;
|
||||
_choices.add(choice);
|
||||
}
|
||||
}
|
||||
|
||||
_blankCount = grouper.blankCount;
|
||||
_errorCount = grouper.errorCount;
|
||||
}
|
||||
|
||||
protected Object[] createMatches() {
|
||||
Object[] a = new Object[_selection.size()];
|
||||
for (int i = 0; i < a.length; i++) {
|
||||
a[i] = _selection.get(i).decoratedValue.value;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.Jsonizable;
|
||||
import com.google.gridworks.browsing.DecoratedValue;
|
||||
|
||||
/**
|
||||
* Store a facet choice that has a decorated value, a count of matched rows,
|
||||
* and a flag of whether it has been selected.
|
||||
*/
|
||||
public class NominalFacetChoice implements Jsonizable {
|
||||
final public DecoratedValue decoratedValue;
|
||||
public int count;
|
||||
public boolean selected;
|
||||
|
||||
public NominalFacetChoice(DecoratedValue decoratedValue) {
|
||||
this.decoratedValue = decoratedValue;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
writer.object();
|
||||
writer.key("v"); decoratedValue.write(writer, options);
|
||||
writer.key("c"); writer.value(count);
|
||||
writer.key("s"); writer.value(selected);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,259 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.filters.AnyRowRecordFilter;
|
||||
import com.google.gridworks.browsing.filters.ExpressionNumberComparisonRowFilter;
|
||||
import com.google.gridworks.browsing.util.ExpressionBasedRowEvaluable;
|
||||
import com.google.gridworks.browsing.util.ExpressionNumericValueBinner;
|
||||
import com.google.gridworks.browsing.util.NumericBinIndex;
|
||||
import com.google.gridworks.browsing.util.NumericBinRecordIndex;
|
||||
import com.google.gridworks.browsing.util.NumericBinRowIndex;
|
||||
import com.google.gridworks.browsing.util.RowEvaluable;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.MetaParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.util.JSONUtilities;
|
||||
|
||||
public class RangeFacet implements Facet {
|
||||
/*
|
||||
* Configuration, from the client side
|
||||
*/
|
||||
protected String _name; // name of facet
|
||||
protected String _expression; // expression to compute numeric value(s) per row
|
||||
protected String _columnName; // column to base expression on, if any
|
||||
|
||||
protected double _from; // the numeric selection
|
||||
protected double _to;
|
||||
|
||||
protected boolean _selectNumeric; // whether the numeric selection applies, default true
|
||||
protected boolean _selectNonNumeric;
|
||||
protected boolean _selectBlank;
|
||||
protected boolean _selectError;
|
||||
|
||||
/*
|
||||
* Derived configuration data
|
||||
*/
|
||||
protected int _cellIndex;
|
||||
protected Evaluable _eval;
|
||||
protected String _errorMessage;
|
||||
protected boolean _selected; // false if we're certain that all rows will match
|
||||
// and there isn't any filtering to do
|
||||
|
||||
/*
|
||||
* Computed data, to return to the client side
|
||||
*/
|
||||
protected double _min;
|
||||
protected double _max;
|
||||
protected double _step;
|
||||
protected int[] _baseBins;
|
||||
protected int[] _bins;
|
||||
|
||||
protected int _baseNumericCount;
|
||||
protected int _baseNonNumericCount;
|
||||
protected int _baseBlankCount;
|
||||
protected int _baseErrorCount;
|
||||
|
||||
protected int _numericCount;
|
||||
protected int _nonNumericCount;
|
||||
protected int _blankCount;
|
||||
protected int _errorCount;
|
||||
|
||||
public RangeFacet() {
|
||||
}
|
||||
|
||||
protected static final String MIN = "min";
|
||||
protected static final String MAX = "max";
|
||||
protected static final String TO = "to";
|
||||
protected static final String FROM = "from";
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("name"); writer.value(_name);
|
||||
writer.key("expression"); writer.value(_expression);
|
||||
writer.key("columnName"); writer.value(_columnName);
|
||||
|
||||
if (_errorMessage != null) {
|
||||
writer.key("error"); writer.value(_errorMessage);
|
||||
} else {
|
||||
if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) {
|
||||
writer.key(MIN); writer.value(_min);
|
||||
writer.key(MAX); writer.value(_max);
|
||||
writer.key("step"); writer.value(_step);
|
||||
|
||||
writer.key("bins"); writer.array();
|
||||
for (int b : _bins) {
|
||||
writer.value(b);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
writer.key("baseBins"); writer.array();
|
||||
for (int b : _baseBins) {
|
||||
writer.value(b);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
writer.key(FROM); writer.value(_from);
|
||||
writer.key(TO); writer.value(_to);
|
||||
}
|
||||
|
||||
writer.key("baseNumericCount"); writer.value(_baseNumericCount);
|
||||
writer.key("baseNonNumericCount"); writer.value(_baseNonNumericCount);
|
||||
writer.key("baseBlankCount"); writer.value(_baseBlankCount);
|
||||
writer.key("baseErrorCount"); writer.value(_baseErrorCount);
|
||||
|
||||
writer.key("numericCount"); writer.value(_numericCount);
|
||||
writer.key("nonNumericCount"); writer.value(_nonNumericCount);
|
||||
writer.key("blankCount"); writer.value(_blankCount);
|
||||
writer.key("errorCount"); writer.value(_errorCount);
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
_name = o.getString("name");
|
||||
_expression = o.getString("expression");
|
||||
_columnName = o.getString("columnName");
|
||||
|
||||
if (_columnName.length() > 0) {
|
||||
Column column = project.columnModel.getColumnByName(_columnName);
|
||||
if (column != null) {
|
||||
_cellIndex = column.getCellIndex();
|
||||
} else {
|
||||
_errorMessage = "No column named " + _columnName;
|
||||
}
|
||||
} else {
|
||||
_cellIndex = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
_eval = MetaParser.parse(_expression);
|
||||
} catch (ParsingException e) {
|
||||
_errorMessage = e.getMessage();
|
||||
}
|
||||
|
||||
if (o.has(FROM) || o.has(TO)) {
|
||||
_from = o.has(FROM) ? o.getDouble(FROM) : _min;
|
||||
_to = o.has(TO) ? o.getDouble(TO) : _max;
|
||||
_selected = true;
|
||||
}
|
||||
|
||||
_selectNumeric = JSONUtilities.getBoolean(o, "selectNumeric", true);
|
||||
_selectNonNumeric = JSONUtilities.getBoolean(o, "selectNonNumeric", true);
|
||||
_selectBlank = JSONUtilities.getBoolean(o, "selectBlank", true);
|
||||
_selectError = JSONUtilities.getBoolean(o, "selectError", true);
|
||||
|
||||
if (!_selectNumeric || !_selectNonNumeric || !_selectBlank || !_selectError) {
|
||||
_selected = true;
|
||||
}
|
||||
}
|
||||
|
||||
public RowFilter getRowFilter(Project project) {
|
||||
if (_eval != null && _errorMessage == null && _selected) {
|
||||
return new ExpressionNumberComparisonRowFilter(
|
||||
getRowEvaluable(project), _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
|
||||
|
||||
protected boolean checkValue(double d) {
|
||||
return d >= _from && d < _to;
|
||||
};
|
||||
};
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordFilter getRecordFilter(Project project) {
|
||||
RowFilter rowFilter = getRowFilter(project);
|
||||
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRows filteredRows) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
RowEvaluable rowEvaluable = getRowEvaluable(project);
|
||||
|
||||
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
|
||||
String key = "numeric-bin:row-based:" + _expression;
|
||||
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
|
||||
if (index == null) {
|
||||
index = new NumericBinRowIndex(project, rowEvaluable);
|
||||
column.setPrecompute(key, index);
|
||||
}
|
||||
|
||||
retrieveDataFromBaseBinIndex(index);
|
||||
|
||||
ExpressionNumericValueBinner binner =
|
||||
new ExpressionNumericValueBinner(rowEvaluable, index);
|
||||
|
||||
filteredRows.accept(project, binner);
|
||||
retrieveDataFromBinner(binner);
|
||||
}
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
RowEvaluable rowEvaluable = getRowEvaluable(project);
|
||||
|
||||
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
|
||||
String key = "numeric-bin:record-based:" + _expression;
|
||||
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
|
||||
if (index == null) {
|
||||
index = new NumericBinRecordIndex(project, rowEvaluable);
|
||||
column.setPrecompute(key, index);
|
||||
}
|
||||
|
||||
retrieveDataFromBaseBinIndex(index);
|
||||
|
||||
ExpressionNumericValueBinner binner =
|
||||
new ExpressionNumericValueBinner(rowEvaluable, index);
|
||||
|
||||
filteredRecords.accept(project, binner);
|
||||
|
||||
retrieveDataFromBinner(binner);
|
||||
}
|
||||
}
|
||||
|
||||
protected RowEvaluable getRowEvaluable(Project project) {
|
||||
return new ExpressionBasedRowEvaluable(_columnName, _cellIndex, _eval);
|
||||
}
|
||||
|
||||
protected void retrieveDataFromBaseBinIndex(NumericBinIndex index) {
|
||||
_min = index.getMin();
|
||||
_max = index.getMax();
|
||||
_step = index.getStep();
|
||||
_baseBins = index.getBins();
|
||||
|
||||
_baseNumericCount = index.getNumericRowCount();
|
||||
_baseNonNumericCount = index.getNonNumericRowCount();
|
||||
_baseBlankCount = index.getBlankRowCount();
|
||||
_baseErrorCount = index.getErrorRowCount();
|
||||
|
||||
if (_selected) {
|
||||
_from = Math.max(_from, _min);
|
||||
_to = Math.min(_to, _max);
|
||||
} else {
|
||||
_from = _min;
|
||||
_to = _max;
|
||||
}
|
||||
}
|
||||
|
||||
protected void retrieveDataFromBinner(ExpressionNumericValueBinner binner) {
|
||||
_bins = binner.bins;
|
||||
_numericCount = binner.numericCount;
|
||||
_nonNumericCount = binner.nonNumericCount;
|
||||
_blankCount = binner.blankCount;
|
||||
_errorCount = binner.errorCount;
|
||||
}
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.awt.BasicStroke;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.awt.image.RenderedImage;
|
||||
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class ScatterplotDrawingRowVisitor implements RowVisitor, RecordVisitor {
|
||||
|
||||
int col_x;
|
||||
int col_y;
|
||||
int dim_x;
|
||||
int dim_y;
|
||||
int rotation;
|
||||
|
||||
double l;
|
||||
double dot;
|
||||
|
||||
double min_x;
|
||||
double max_x;
|
||||
double min_y;
|
||||
double max_y;
|
||||
|
||||
BufferedImage image;
|
||||
Graphics2D g2;
|
||||
|
||||
AffineTransform r;
|
||||
|
||||
public ScatterplotDrawingRowVisitor(
|
||||
int col_x, int col_y, double min_x, double max_x, double min_y, double max_y,
|
||||
int size, int dim_x, int dim_y, int rotation, double dot, Color color)
|
||||
{
|
||||
this.col_x = col_x;
|
||||
this.col_y = col_y;
|
||||
this.min_x = min_x;
|
||||
this.min_y = min_y;
|
||||
this.max_x = max_x;
|
||||
this.max_y = max_y;
|
||||
this.dot = dot;
|
||||
this.dim_x = dim_x;
|
||||
this.dim_y = dim_y;
|
||||
this.rotation = rotation;
|
||||
|
||||
l = (double) size;
|
||||
r = ScatterplotFacet.createRotationMatrix(rotation, l);
|
||||
|
||||
image = new BufferedImage(size, size, BufferedImage.TYPE_4BYTE_ABGR);
|
||||
g2 = (Graphics2D) image.getGraphics();
|
||||
g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
|
||||
g2.setStroke(new BasicStroke(1.0f));
|
||||
|
||||
AffineTransform t = AffineTransform.getTranslateInstance(0, l);
|
||||
t.scale(1, -1);
|
||||
|
||||
g2.setTransform(t);
|
||||
g2.setColor(color);
|
||||
g2.setPaint(color);
|
||||
|
||||
if (r != null) {
|
||||
/*
|
||||
* Fill in the negative quadrants to give a hint of how the plot has been rotated.
|
||||
*/
|
||||
Graphics2D g2r = (Graphics2D) g2.create();
|
||||
g2r.transform(r);
|
||||
|
||||
g2r.setPaint(Color.lightGray);
|
||||
g2r.fillRect(-size, 0, size, size);
|
||||
g2r.fillRect(0, -size, size, size);
|
||||
g2r.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public void setColor(Color color) {
|
||||
g2.setColor(color);
|
||||
g2.setPaint(color);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
Cell cellx = row.getCell(col_x);
|
||||
Cell celly = row.getCell(col_y);
|
||||
if ((cellx != null && cellx.value != null && cellx.value instanceof Number) &&
|
||||
(celly != null && celly.value != null && celly.value instanceof Number))
|
||||
{
|
||||
double xv = ((Number) cellx.value).doubleValue();
|
||||
double yv = ((Number) celly.value).doubleValue();
|
||||
|
||||
Point2D.Double p = new Point2D.Double(xv,yv);
|
||||
|
||||
p = ScatterplotFacet.translateCoordinates(
|
||||
p, min_x, max_x, min_y, max_y, dim_x, dim_y, l, r);
|
||||
|
||||
g2.fill(new Rectangle2D.Double(p.x - dot / 2, p.y - dot / 2, dot, dot));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, Record record) {
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
visit(project, r, project.rows.get(r));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public RenderedImage getImage() {
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,452 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.awt.image.RenderedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.filters.AnyRowRecordFilter;
|
||||
import com.google.gridworks.browsing.filters.DualExpressionsNumberComparisonRowFilter;
|
||||
import com.google.gridworks.browsing.util.ExpressionBasedRowEvaluable;
|
||||
import com.google.gridworks.browsing.util.NumericBinIndex;
|
||||
import com.google.gridworks.browsing.util.NumericBinRecordIndex;
|
||||
import com.google.gridworks.browsing.util.NumericBinRowIndex;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.MetaParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class ScatterplotFacet implements Facet {
|
||||
|
||||
public static final int LIN = 0;
|
||||
public static final int LOG = 1;
|
||||
|
||||
public static final int NO_ROTATION = 0;
|
||||
public static final int ROTATE_CW = 1;
|
||||
public static final int ROTATE_CCW = 2;
|
||||
|
||||
/*
|
||||
* Configuration, from the client side
|
||||
*/
|
||||
protected String name; // name of facet
|
||||
|
||||
protected String expression_x; // expression to compute the x numeric value(s) per row
|
||||
protected String expression_y; // expression to compute the y numeric value(s) per row
|
||||
protected String columnName_x; // column to base the x expression on, if any
|
||||
protected String columnName_y; // column to base the y expression on, if any
|
||||
|
||||
protected int size;
|
||||
protected int dim_x;
|
||||
protected int dim_y;
|
||||
protected int rotation;
|
||||
|
||||
protected double l;
|
||||
protected double dot;
|
||||
|
||||
protected String image;
|
||||
|
||||
protected String color_str;
|
||||
protected Color color;
|
||||
|
||||
protected double from_x; // the numeric selection for the x axis, from 0 to 1
|
||||
protected double to_x;
|
||||
protected double from_y; // the numeric selection for the y axis, from 0 to 1
|
||||
protected double to_y;
|
||||
|
||||
/*
|
||||
* Derived configuration data
|
||||
*/
|
||||
protected int columnIndex_x;
|
||||
protected int columnIndex_y;
|
||||
protected Evaluable eval_x;
|
||||
protected Evaluable eval_y;
|
||||
protected String errorMessage_x;
|
||||
protected String errorMessage_y;
|
||||
|
||||
protected double min_x;
|
||||
protected double max_x;
|
||||
protected double min_y;
|
||||
protected double max_y;
|
||||
protected AffineTransform t;
|
||||
|
||||
protected boolean selected; // false if we're certain that all rows will match
|
||||
// and there isn't any filtering to do
|
||||
|
||||
public static final String NAME = "name";
|
||||
public static final String IMAGE = "image";
|
||||
public static final String COLOR = "color";
|
||||
public static final String BASE_COLOR = "base_color";
|
||||
public static final String SIZE = "l";
|
||||
public static final String ROTATION = "r";
|
||||
public static final String DOT = "dot";
|
||||
public static final String DIM_X = "dim_x";
|
||||
public static final String DIM_Y = "dim_y";
|
||||
|
||||
public static final String X_COLUMN_NAME = "cx";
|
||||
public static final String X_EXPRESSION = "ex";
|
||||
public static final String MIN_X = "min_x";
|
||||
public static final String MAX_X = "max_x";
|
||||
public static final String TO_X = "to_x";
|
||||
public static final String FROM_X = "from_x";
|
||||
public static final String ERROR_X = "error_x";
|
||||
|
||||
public static final String Y_COLUMN_NAME = "cy";
|
||||
public static final String Y_EXPRESSION = "ey";
|
||||
public static final String MIN_Y = "min_y";
|
||||
public static final String MAX_Y = "max_y";
|
||||
public static final String TO_Y = "to_y";
|
||||
public static final String FROM_Y = "from_y";
|
||||
public static final String ERROR_Y = "error_y";
|
||||
|
||||
private static final boolean IMAGE_URI = false;
|
||||
|
||||
public static String EMPTY_IMAGE;
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("scatterplot_facet");
|
||||
|
||||
static {
|
||||
try {
|
||||
EMPTY_IMAGE = serializeImage(new BufferedImage(1, 1, BufferedImage.TYPE_4BYTE_ABGR));
|
||||
} catch (IOException e) {
|
||||
EMPTY_IMAGE = "";
|
||||
}
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
|
||||
writer.object();
|
||||
|
||||
writer.key(NAME); writer.value(name);
|
||||
writer.key(X_COLUMN_NAME); writer.value(columnName_x);
|
||||
writer.key(X_EXPRESSION); writer.value(expression_x);
|
||||
writer.key(Y_COLUMN_NAME); writer.value(columnName_y);
|
||||
writer.key(Y_EXPRESSION); writer.value(expression_y);
|
||||
writer.key(SIZE); writer.value(size);
|
||||
writer.key(DOT); writer.value(dot);
|
||||
writer.key(ROTATION); writer.value(rotation);
|
||||
writer.key(DIM_X); writer.value(dim_x);
|
||||
writer.key(DIM_Y); writer.value(dim_y);
|
||||
writer.key(COLOR); writer.value(color_str);
|
||||
|
||||
if (IMAGE_URI) {
|
||||
writer.key(IMAGE); writer.value(image);
|
||||
}
|
||||
|
||||
if (errorMessage_x != null) {
|
||||
writer.key(ERROR_X); writer.value(errorMessage_x);
|
||||
} else {
|
||||
if (!Double.isInfinite(min_x) && !Double.isInfinite(max_x)) {
|
||||
writer.key(FROM_X); writer.value(from_x);
|
||||
writer.key(TO_X); writer.value(to_x);
|
||||
}
|
||||
}
|
||||
|
||||
if (errorMessage_y != null) {
|
||||
writer.key(ERROR_Y); writer.value(errorMessage_y);
|
||||
} else {
|
||||
if (!Double.isInfinite(min_y) && !Double.isInfinite(max_y)) {
|
||||
writer.key(FROM_Y); writer.value(from_y);
|
||||
writer.key(TO_Y); writer.value(to_y);
|
||||
}
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
name = o.getString(NAME);
|
||||
l = size = (o.has(SIZE)) ? o.getInt(SIZE) : 100;
|
||||
dot = (o.has(DOT)) ? o.getInt(DOT) : 0.5d;
|
||||
|
||||
dim_x = (o.has(DIM_X)) ? getAxisDim(o.getString(DIM_X)) : LIN;
|
||||
if (o.has(FROM_X) && o.has(TO_X)) {
|
||||
from_x = o.getDouble(FROM_X);
|
||||
to_x = o.getDouble(TO_X);
|
||||
selected = true;
|
||||
} else {
|
||||
from_x = 0;
|
||||
to_x = 1;
|
||||
}
|
||||
|
||||
dim_y = (o.has(DIM_Y)) ? getAxisDim(o.getString(DIM_Y)) : LIN;
|
||||
if (o.has(FROM_Y) && o.has(TO_Y)) {
|
||||
from_y = o.getDouble(FROM_Y);
|
||||
to_y = o.getDouble(TO_Y);
|
||||
selected = true;
|
||||
} else {
|
||||
from_y = 0;
|
||||
to_y = 1;
|
||||
}
|
||||
|
||||
rotation = (o.has(ROTATION)) ? getRotation(o.getString(ROTATION)) : NO_ROTATION;
|
||||
t = createRotationMatrix(rotation, l);
|
||||
|
||||
color_str = (o.has(COLOR)) ? o.getString(COLOR) : "000000";
|
||||
color = new Color(Integer.parseInt(color_str,16));
|
||||
|
||||
columnName_x = o.getString(X_COLUMN_NAME);
|
||||
expression_x = o.getString(X_EXPRESSION);
|
||||
|
||||
if (columnName_x.length() > 0) {
|
||||
Column x_column = project.columnModel.getColumnByName(columnName_x);
|
||||
if (x_column != null) {
|
||||
columnIndex_x = x_column.getCellIndex();
|
||||
|
||||
NumericBinIndex index_x = ScatterplotFacet.getBinIndex(project, x_column, eval_x, expression_x);
|
||||
min_x = index_x.getMin();
|
||||
max_x = index_x.getMax();
|
||||
} else {
|
||||
errorMessage_x = "No column named " + columnName_x;
|
||||
}
|
||||
} else {
|
||||
columnIndex_x = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
eval_x = MetaParser.parse(expression_x);
|
||||
} catch (ParsingException e) {
|
||||
errorMessage_x = e.getMessage();
|
||||
}
|
||||
|
||||
columnName_y = o.getString(Y_COLUMN_NAME);
|
||||
expression_y = o.getString(Y_EXPRESSION);
|
||||
|
||||
if (columnName_y.length() > 0) {
|
||||
Column y_column = project.columnModel.getColumnByName(columnName_y);
|
||||
if (y_column != null) {
|
||||
columnIndex_y = y_column.getCellIndex();
|
||||
|
||||
NumericBinIndex index_y = ScatterplotFacet.getBinIndex(project, y_column, eval_y, expression_y);
|
||||
min_y = index_y.getMin();
|
||||
max_y = index_y.getMax();
|
||||
} else {
|
||||
errorMessage_y = "No column named " + columnName_y;
|
||||
}
|
||||
} else {
|
||||
columnIndex_y = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
eval_y = MetaParser.parse(expression_y);
|
||||
} catch (ParsingException e) {
|
||||
errorMessage_y = e.getMessage();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public RowFilter getRowFilter(Project project) {
|
||||
if (selected &&
|
||||
eval_x != null && errorMessage_x == null &&
|
||||
eval_y != null && errorMessage_y == null)
|
||||
{
|
||||
return new DualExpressionsNumberComparisonRowFilter(
|
||||
eval_x, columnName_x, columnIndex_x, eval_y, columnName_y, columnIndex_y) {
|
||||
|
||||
double from_x_pixels = from_x * l;
|
||||
double to_x_pixels = to_x * l;
|
||||
double from_y_pixels = from_y * l;
|
||||
double to_y_pixels = to_y * l;
|
||||
|
||||
protected boolean checkValues(double x, double y) {
|
||||
Point2D.Double p = new Point2D.Double(x,y);
|
||||
p = translateCoordinates(p, min_x, max_x, min_y, max_y, dim_x, dim_y, l, t);
|
||||
return p.x >= from_x_pixels && p.x <= to_x_pixels && p.y >= from_y_pixels && p.y <= to_y_pixels;
|
||||
};
|
||||
};
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordFilter getRecordFilter(Project project) {
|
||||
RowFilter rowFilter = getRowFilter(project);
|
||||
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRows filteredRows) {
|
||||
if (eval_x != null && eval_y != null && errorMessage_x == null && errorMessage_y == null) {
|
||||
Column column_x = project.columnModel.getColumnByCellIndex(columnIndex_x);
|
||||
NumericBinIndex index_x = getBinIndex(project, column_x, eval_x, expression_x, "row-based");
|
||||
|
||||
Column column_y = project.columnModel.getColumnByCellIndex(columnIndex_y);
|
||||
NumericBinIndex index_y = getBinIndex(project, column_y, eval_y, expression_y, "row-based");
|
||||
|
||||
retrieveDataFromBinIndices(index_x, index_y);
|
||||
|
||||
if (IMAGE_URI) {
|
||||
if (index_x.isNumeric() && index_y.isNumeric()) {
|
||||
ScatterplotDrawingRowVisitor drawer = new ScatterplotDrawingRowVisitor(
|
||||
columnIndex_x, columnIndex_y, min_x, max_x, min_y, max_y,
|
||||
size, dim_x, dim_y, rotation, dot, color
|
||||
);
|
||||
filteredRows.accept(project, drawer);
|
||||
|
||||
try {
|
||||
image = serializeImage(drawer.getImage());
|
||||
} catch (IOException e) {
|
||||
logger.warn("Exception caught while generating the image", e);
|
||||
}
|
||||
} else {
|
||||
image = EMPTY_IMAGE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords) {
|
||||
if (eval_x != null && eval_y != null && errorMessage_x == null && errorMessage_y == null) {
|
||||
Column column_x = project.columnModel.getColumnByCellIndex(columnIndex_x);
|
||||
NumericBinIndex index_x = getBinIndex(project, column_x, eval_x, expression_x, "record-based");
|
||||
|
||||
Column column_y = project.columnModel.getColumnByCellIndex(columnIndex_y);
|
||||
NumericBinIndex index_y = getBinIndex(project, column_y, eval_y, expression_y, "record-based");
|
||||
|
||||
retrieveDataFromBinIndices(index_x, index_y);
|
||||
|
||||
if (IMAGE_URI) {
|
||||
if (index_x.isNumeric() && index_y.isNumeric()) {
|
||||
ScatterplotDrawingRowVisitor drawer = new ScatterplotDrawingRowVisitor(
|
||||
columnIndex_x, columnIndex_y, min_x, max_x, min_y, max_y,
|
||||
size, dim_x, dim_y, rotation, dot, color
|
||||
);
|
||||
filteredRecords.accept(project, drawer);
|
||||
|
||||
try {
|
||||
image = serializeImage(drawer.getImage());
|
||||
} catch (IOException e) {
|
||||
logger.warn("Exception caught while generating the image", e);
|
||||
}
|
||||
} else {
|
||||
image = EMPTY_IMAGE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void retrieveDataFromBinIndices(NumericBinIndex index_x, NumericBinIndex index_y) {
|
||||
min_x = index_x.getMin();
|
||||
max_x = index_x.getMax();
|
||||
|
||||
min_y = index_y.getMin();
|
||||
max_y = index_y.getMax();
|
||||
}
|
||||
|
||||
public static String serializeImage(RenderedImage image) throws IOException {
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream(4096);
|
||||
ImageIO.write(image, "png", output);
|
||||
output.close();
|
||||
String encoded = Base64.encodeBase64String(output.toByteArray());
|
||||
String url = "data:image/png;base64," + encoded;
|
||||
return url;
|
||||
}
|
||||
|
||||
public static int getAxisDim(String type) {
|
||||
return ("log".equals(type.toLowerCase())) ? LOG : LIN;
|
||||
}
|
||||
|
||||
public static int getRotation(String rotation) {
|
||||
rotation = rotation.toLowerCase();
|
||||
if ("cw".equals(rotation) || "right".equals(rotation)) {
|
||||
return ScatterplotFacet.ROTATE_CW;
|
||||
} else if ("ccw".equals(rotation) || "left".equals(rotation)) {
|
||||
return ScatterplotFacet.ROTATE_CCW;
|
||||
} else {
|
||||
return NO_ROTATION;
|
||||
}
|
||||
}
|
||||
|
||||
public static NumericBinIndex getBinIndex(Project project, Column column, Evaluable eval, String expression) {
|
||||
return getBinIndex(project, column, eval, expression, "row-based");
|
||||
}
|
||||
|
||||
public static NumericBinIndex getBinIndex(Project project, Column column, Evaluable eval, String expression, String mode) {
|
||||
String key = "numeric-bin:" + mode + ":" + expression;
|
||||
if (eval == null) {
|
||||
try {
|
||||
eval = MetaParser.parse(expression);
|
||||
} catch (ParsingException e) {
|
||||
logger.warn("Error parsing expression",e);
|
||||
}
|
||||
}
|
||||
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
|
||||
if (index == null) {
|
||||
index = "row-based".equals(mode) ?
|
||||
new NumericBinRowIndex(project, new ExpressionBasedRowEvaluable(column.getName(), column.getCellIndex(), eval)) :
|
||||
new NumericBinRecordIndex(project, new ExpressionBasedRowEvaluable(column.getName(), column.getCellIndex(), eval));
|
||||
|
||||
column.setPrecompute(key, index);
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
private static double s_rotateScale = 1 / Math.sqrt(2.0);
|
||||
|
||||
public static AffineTransform createRotationMatrix(int rotation, double l) {
|
||||
if (rotation == ScatterplotFacet.ROTATE_CW) {
|
||||
AffineTransform t = AffineTransform.getTranslateInstance(0, l / 2);
|
||||
t.scale(s_rotateScale, s_rotateScale);
|
||||
t.rotate(-Math.PI / 4);
|
||||
return t;
|
||||
} else if (rotation == ScatterplotFacet.ROTATE_CCW) {
|
||||
AffineTransform t = AffineTransform.getTranslateInstance(l / 2, 0);
|
||||
t.scale(s_rotateScale, s_rotateScale);
|
||||
t.rotate(Math.PI / 4);
|
||||
return t;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static Point2D.Double translateCoordinates(
|
||||
Point2D.Double p,
|
||||
double min_x, double max_x, double min_y, double max_y,
|
||||
int dim_x, int dim_y, double l, AffineTransform t) {
|
||||
|
||||
double x = p.x;
|
||||
double y = p.y;
|
||||
|
||||
double relative_x = x - min_x;
|
||||
double range_x = max_x - min_x;
|
||||
if (dim_x == ScatterplotFacet.LOG) {
|
||||
x = Math.log10(relative_x + 1) * l / Math.log10(range_x + 1);
|
||||
} else {
|
||||
x = relative_x * l / range_x;
|
||||
}
|
||||
|
||||
double relative_y = y - min_y;
|
||||
double range_y = max_y - min_y;
|
||||
if (dim_y == ScatterplotFacet.LOG) {
|
||||
y = Math.log10(relative_y + 1) * l / Math.log10(range_y + 1);
|
||||
} else {
|
||||
y = relative_y * l / range_y;
|
||||
}
|
||||
|
||||
p.x = x;
|
||||
p.y = y;
|
||||
if (t != null) {
|
||||
t.transform(p, p);
|
||||
}
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,120 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.filters.AnyRowRecordFilter;
|
||||
import com.google.gridworks.browsing.filters.ExpressionStringComparisonRowFilter;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.gel.ast.VariableExpr;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class TextSearchFacet implements Facet {
|
||||
/*
|
||||
* Configuration
|
||||
*/
|
||||
protected String _name;
|
||||
protected String _columnName;
|
||||
protected String _query;
|
||||
protected String _mode;
|
||||
protected boolean _caseSensitive;
|
||||
|
||||
/*
|
||||
* Derived configuration
|
||||
*/
|
||||
protected int _cellIndex;
|
||||
protected Pattern _pattern;
|
||||
|
||||
public TextSearchFacet() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("name"); writer.value(_name);
|
||||
writer.key("columnName"); writer.value(_columnName);
|
||||
writer.key("query"); writer.value(_query);
|
||||
writer.key("mode"); writer.value(_mode);
|
||||
writer.key("caseSensitive"); writer.value(_caseSensitive);
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
_name = o.getString("name");
|
||||
_columnName = o.getString("columnName");
|
||||
|
||||
_cellIndex = project.columnModel.getColumnByName(_columnName).getCellIndex();
|
||||
|
||||
if (!o.isNull("query")) {
|
||||
_query = o.getString("query");
|
||||
}
|
||||
|
||||
_mode = o.getString("mode");
|
||||
_caseSensitive = o.getBoolean("caseSensitive");
|
||||
if (_query != null) {
|
||||
if ("regex".equals(_mode)) {
|
||||
try {
|
||||
_pattern = Pattern.compile(
|
||||
_query,
|
||||
_caseSensitive ? 0 : Pattern.CASE_INSENSITIVE);
|
||||
} catch (java.util.regex.PatternSyntaxException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
} else if (!_caseSensitive) {
|
||||
_query = _query.toLowerCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public RowFilter getRowFilter(Project project) {
|
||||
if (_query == null || _query.length() == 0) {
|
||||
return null;
|
||||
} else if ("regex".equals(_mode) && _pattern == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Evaluable eval = new VariableExpr("value");
|
||||
|
||||
if ("regex".equals(_mode)) {
|
||||
return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) {
|
||||
protected boolean checkValue(String s) {
|
||||
return _pattern.matcher(s).find();
|
||||
};
|
||||
};
|
||||
} else {
|
||||
return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) {
|
||||
protected boolean checkValue(String s) {
|
||||
return (_caseSensitive ? s : s.toLowerCase()).contains(_query);
|
||||
};
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordFilter getRecordFilter(Project project) {
|
||||
RowFilter rowFilter = getRowFilter(project);
|
||||
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void computeChoices(Project project, FilteredRows filteredRows) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords) {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
||||
@ -1,201 +0,0 @@
|
||||
package com.google.gridworks.browsing.facets;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.filters.ExpressionTimeComparisonRowFilter;
|
||||
import com.google.gridworks.browsing.util.ExpressionTimeValueBinner;
|
||||
import com.google.gridworks.browsing.util.RowEvaluable;
|
||||
import com.google.gridworks.browsing.util.TimeBinIndex;
|
||||
import com.google.gridworks.browsing.util.TimeBinRecordIndex;
|
||||
import com.google.gridworks.browsing.util.TimeBinRowIndex;
|
||||
import com.google.gridworks.expr.MetaParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.util.JSONUtilities;
|
||||
|
||||
public class TimeRangeFacet extends RangeFacet {
|
||||
|
||||
protected boolean _selectTime; // whether the time selection applies, default true
|
||||
protected boolean _selectNonTime;
|
||||
|
||||
protected int _baseTimeCount;
|
||||
protected int _baseNonTimeCount;
|
||||
|
||||
protected int _timeCount;
|
||||
protected int _nonTimeCount;
|
||||
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("name"); writer.value(_name);
|
||||
writer.key("expression"); writer.value(_expression);
|
||||
writer.key("columnName"); writer.value(_columnName);
|
||||
|
||||
if (_errorMessage != null) {
|
||||
writer.key("error"); writer.value(_errorMessage);
|
||||
} else {
|
||||
if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) {
|
||||
writer.key(MIN); writer.value(_min);
|
||||
writer.key(MAX); writer.value(_max);
|
||||
writer.key("step"); writer.value(_step);
|
||||
|
||||
writer.key("bins"); writer.array();
|
||||
for (int b : _bins) {
|
||||
writer.value(b);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
writer.key("baseBins"); writer.array();
|
||||
for (int b : _baseBins) {
|
||||
writer.value(b);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
writer.key(FROM); writer.value(_from);
|
||||
writer.key(TO); writer.value(_to);
|
||||
}
|
||||
|
||||
writer.key("baseTimeCount"); writer.value(_baseTimeCount);
|
||||
writer.key("baseNonTimeCount"); writer.value(_baseNonTimeCount);
|
||||
writer.key("baseBlankCount"); writer.value(_baseBlankCount);
|
||||
writer.key("baseErrorCount"); writer.value(_baseErrorCount);
|
||||
|
||||
writer.key("timeCount"); writer.value(_timeCount);
|
||||
writer.key("nonTimeCount"); writer.value(_nonTimeCount);
|
||||
writer.key("blankCount"); writer.value(_blankCount);
|
||||
writer.key("errorCount"); writer.value(_errorCount);
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
_name = o.getString("name");
|
||||
_expression = o.getString("expression");
|
||||
_columnName = o.getString("columnName");
|
||||
|
||||
if (_columnName.length() > 0) {
|
||||
Column column = project.columnModel.getColumnByName(_columnName);
|
||||
if (column != null) {
|
||||
_cellIndex = column.getCellIndex();
|
||||
} else {
|
||||
_errorMessage = "No column named " + _columnName;
|
||||
}
|
||||
} else {
|
||||
_cellIndex = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
_eval = MetaParser.parse(_expression);
|
||||
} catch (ParsingException e) {
|
||||
_errorMessage = e.getMessage();
|
||||
}
|
||||
|
||||
if (o.has(FROM) || o.has(TO)) {
|
||||
_from = o.has(FROM) ? o.getDouble(FROM) : _min;
|
||||
_to = o.has(TO) ? o.getDouble(TO) : _max;
|
||||
_selected = true;
|
||||
}
|
||||
|
||||
_selectTime = JSONUtilities.getBoolean(o, "selectTime", true);
|
||||
_selectNonTime = JSONUtilities.getBoolean(o, "selectNonTime", true);
|
||||
_selectBlank = JSONUtilities.getBoolean(o, "selectBlank", true);
|
||||
_selectError = JSONUtilities.getBoolean(o, "selectError", true);
|
||||
|
||||
if (!_selectTime || !_selectNonTime || !_selectBlank || !_selectError) {
|
||||
_selected = true;
|
||||
}
|
||||
}
|
||||
|
||||
public RowFilter getRowFilter(Project project) {
|
||||
if (_eval != null && _errorMessage == null && _selected) {
|
||||
return new ExpressionTimeComparisonRowFilter(
|
||||
getRowEvaluable(project), _selectTime, _selectNonTime, _selectBlank, _selectError) {
|
||||
|
||||
protected boolean checkValue(long t) {
|
||||
return t >= _from && t < _to;
|
||||
};
|
||||
};
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRows filteredRows) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
RowEvaluable rowEvaluable = getRowEvaluable(project);
|
||||
|
||||
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
|
||||
String key = "time-bin:row-based:" + _expression;
|
||||
TimeBinIndex index = (TimeBinIndex) column.getPrecompute(key);
|
||||
if (index == null) {
|
||||
index = new TimeBinRowIndex(project, rowEvaluable);
|
||||
column.setPrecompute(key, index);
|
||||
}
|
||||
|
||||
retrieveDataFromBaseBinIndex(index);
|
||||
|
||||
ExpressionTimeValueBinner binner = new ExpressionTimeValueBinner(rowEvaluable, index);
|
||||
|
||||
filteredRows.accept(project, binner);
|
||||
retrieveDataFromBinner(binner);
|
||||
}
|
||||
}
|
||||
|
||||
public void computeChoices(Project project, FilteredRecords filteredRecords) {
|
||||
if (_eval != null && _errorMessage == null) {
|
||||
RowEvaluable rowEvaluable = getRowEvaluable(project);
|
||||
|
||||
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
|
||||
String key = "time-bin:record-based:" + _expression;
|
||||
TimeBinIndex index = (TimeBinIndex) column.getPrecompute(key);
|
||||
if (index == null) {
|
||||
index = new TimeBinRecordIndex(project, rowEvaluable);
|
||||
column.setPrecompute(key, index);
|
||||
}
|
||||
|
||||
retrieveDataFromBaseBinIndex(index);
|
||||
|
||||
ExpressionTimeValueBinner binner = new ExpressionTimeValueBinner(rowEvaluable, index);
|
||||
|
||||
filteredRecords.accept(project, binner);
|
||||
|
||||
retrieveDataFromBinner(binner);
|
||||
}
|
||||
}
|
||||
|
||||
protected void retrieveDataFromBaseBinIndex(TimeBinIndex index) {
|
||||
_min = index.getMin();
|
||||
_max = index.getMax();
|
||||
_step = index.getStep();
|
||||
_baseBins = index.getBins();
|
||||
|
||||
_baseTimeCount = index.getTimeRowCount();
|
||||
_baseNonTimeCount = index.getNonTimeRowCount();
|
||||
_baseBlankCount = index.getBlankRowCount();
|
||||
_baseErrorCount = index.getErrorRowCount();
|
||||
|
||||
if (_selected) {
|
||||
_from = Math.max(_from, _min);
|
||||
_to = Math.min(_to, _max);
|
||||
} else {
|
||||
_from = _min;
|
||||
_to = _max;
|
||||
}
|
||||
}
|
||||
|
||||
protected void retrieveDataFromBinner(ExpressionTimeValueBinner binner) {
|
||||
_bins = binner.bins;
|
||||
_timeCount = binner.timeCount;
|
||||
_nonTimeCount = binner.nonTimeCount;
|
||||
_blankCount = binner.blankCount;
|
||||
_errorCount = binner.errorCount;
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
public class AllRowsRecordFilter implements RecordFilter {
|
||||
final protected RowFilter _rowFilter;
|
||||
|
||||
public AllRowsRecordFilter(RowFilter rowFilter) {
|
||||
_rowFilter = rowFilter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean filterRecord(Project project, Record record) {
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
if (!_rowFilter.filterRow(project, r, project.rows.get(r))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
public class AnyRowRecordFilter implements RecordFilter {
|
||||
final protected RowFilter _rowFilter;
|
||||
|
||||
public AnyRowRecordFilter(RowFilter rowFilter) {
|
||||
_rowFilter = rowFilter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean filterRecord(Project project, Record record) {
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
if (_rowFilter.filterRow(project, r, project.rows.get(r))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Judge if a row matches by evaluating two given expressions on the row, based on two different columns
|
||||
* and checking the results. It's a match if the result satisfies some numeric comparisons.
|
||||
*/
|
||||
abstract public class DualExpressionsNumberComparisonRowFilter implements RowFilter {
|
||||
|
||||
final protected Evaluable _x_evaluable;
|
||||
final protected String _x_columnName;
|
||||
final protected int _x_cellIndex;
|
||||
final protected Evaluable _y_evaluable;
|
||||
final protected String _y_columnName;
|
||||
final protected int _y_cellIndex;
|
||||
|
||||
public DualExpressionsNumberComparisonRowFilter (
|
||||
Evaluable x_evaluable,
|
||||
String x_columnName,
|
||||
int x_cellIndex,
|
||||
Evaluable y_evaluable,
|
||||
String y_columnName,
|
||||
int y_cellIndex
|
||||
) {
|
||||
_x_evaluable = x_evaluable;
|
||||
_x_columnName = x_columnName;
|
||||
_x_cellIndex = x_cellIndex;
|
||||
_y_evaluable = y_evaluable;
|
||||
_y_columnName = y_columnName;
|
||||
_y_cellIndex = y_cellIndex;
|
||||
}
|
||||
|
||||
public boolean filterRow(Project project, int rowIndex, Row row) {
|
||||
Cell x_cell = _x_cellIndex < 0 ? null : row.getCell(_x_cellIndex);
|
||||
Properties x_bindings = ExpressionUtils.createBindings(project);
|
||||
ExpressionUtils.bind(x_bindings, row, rowIndex, _x_columnName, x_cell);
|
||||
Object x_value = _x_evaluable.evaluate(x_bindings);
|
||||
|
||||
Cell y_cell = _y_cellIndex < 0 ? null : row.getCell(_y_cellIndex);
|
||||
Properties y_bindings = ExpressionUtils.createBindings(project);
|
||||
ExpressionUtils.bind(y_bindings, row, rowIndex, _y_columnName, y_cell);
|
||||
Object y_value = _y_evaluable.evaluate(y_bindings);
|
||||
|
||||
if (x_value != null && y_value != null) {
|
||||
if (x_value.getClass().isArray() || y_value.getClass().isArray()) {
|
||||
return false;
|
||||
} else if (x_value instanceof Collection<?> || y_value instanceof Collection<?>) {
|
||||
return false;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
return checkValue(x_value,y_value);
|
||||
}
|
||||
|
||||
protected boolean checkValue(Object vx, Object vy) {
|
||||
if (ExpressionUtils.isError(vx) || ExpressionUtils.isError(vy)) {
|
||||
return false;
|
||||
} else if (ExpressionUtils.isNonBlankData(vx) && ExpressionUtils.isNonBlankData(vy)) {
|
||||
if (vx instanceof Number && vy instanceof Number) {
|
||||
double dx = ((Number) vx).doubleValue();
|
||||
double dy = ((Number) vy).doubleValue();
|
||||
return (!Double.isInfinite(dx) &&
|
||||
!Double.isNaN(dx) &&
|
||||
!Double.isInfinite(dy) &&
|
||||
!Double.isNaN(dy) &&
|
||||
checkValues(dx,dy));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
abstract protected boolean checkValues(double dx, double dy);
|
||||
}
|
||||
@ -1,164 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Judge if a row matches by evaluating a given expression on the row, based on a particular
|
||||
* column, and checking the result. It's a match if the result is any one of a given list of
|
||||
* values, or if the result is blank or error and we want blank or error values.
|
||||
*/
|
||||
public class ExpressionEqualRowFilter implements RowFilter {
|
||||
final protected Evaluable _evaluable; // the expression to evaluate
|
||||
|
||||
final protected String _columnName;
|
||||
final protected int _cellIndex; // the expression is based on this column;
|
||||
// -1 if based on no column in particular,
|
||||
// for expression such as "row.starred".
|
||||
|
||||
final protected Object[] _matches;
|
||||
final protected boolean _selectBlank;
|
||||
final protected boolean _selectError;
|
||||
final protected boolean _invert;
|
||||
|
||||
public ExpressionEqualRowFilter(
|
||||
Evaluable evaluable,
|
||||
String columnName,
|
||||
int cellIndex,
|
||||
Object[] matches,
|
||||
boolean selectBlank,
|
||||
boolean selectError,
|
||||
boolean invert
|
||||
) {
|
||||
_evaluable = evaluable;
|
||||
_columnName = columnName;
|
||||
_cellIndex = cellIndex;
|
||||
_matches = matches;
|
||||
_selectBlank = selectBlank;
|
||||
_selectError = selectError;
|
||||
_invert = invert;
|
||||
}
|
||||
|
||||
public boolean filterRow(Project project, int rowIndex, Row row) {
|
||||
return _invert ?
|
||||
internalInvertedFilterRow(project, rowIndex, row) :
|
||||
internalFilterRow(project, rowIndex, row);
|
||||
}
|
||||
|
||||
public boolean internalFilterRow(Project project, int rowIndex, Row row) {
|
||||
Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
|
||||
|
||||
Object value = _evaluable.evaluate(bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
if (testValue(v)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
if (testValue(v)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else if (value instanceof JSONArray) {
|
||||
JSONArray a = (JSONArray) value;
|
||||
int l = a.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
try {
|
||||
if (testValue(a.get(i))) {
|
||||
return true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
return testValue(value);
|
||||
}
|
||||
|
||||
public boolean internalInvertedFilterRow(Project project, int rowIndex, Row row) {
|
||||
Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
|
||||
|
||||
Object value = _evaluable.evaluate(bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
if (testValue(v)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
if (testValue(v)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} else if (value instanceof JSONArray) {
|
||||
JSONArray a = (JSONArray) value;
|
||||
int l = a.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
try {
|
||||
if (testValue(a.get(i))) {
|
||||
return false;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
return !testValue(value);
|
||||
}
|
||||
|
||||
protected boolean testValue(Object v) {
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
return _selectError;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
for (Object match : _matches) {
|
||||
if (testValue(v, match)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
return _selectBlank;
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean testValue(Object v, Object match) {
|
||||
return (v instanceof Number && match instanceof Number) ?
|
||||
((Number) match).doubleValue() == ((Number) v).doubleValue() :
|
||||
match.equals(v);
|
||||
}
|
||||
}
|
||||
@ -1,102 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.util.RowEvaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Judge if a row matches by evaluating a given expression on the row, based on a particular
|
||||
* column, and checking the result. It's a match if the result satisfies some numeric comparisons,
|
||||
* or if the result is non-numeric or blank or error and we want non-numeric or blank or error
|
||||
* values.
|
||||
*/
|
||||
abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
|
||||
final protected RowEvaluable _rowEvaluable;
|
||||
final protected boolean _selectNumeric;
|
||||
final protected boolean _selectNonNumeric;
|
||||
final protected boolean _selectBlank;
|
||||
final protected boolean _selectError;
|
||||
|
||||
public ExpressionNumberComparisonRowFilter(
|
||||
RowEvaluable rowEvaluable,
|
||||
boolean selectNumeric,
|
||||
boolean selectNonNumeric,
|
||||
boolean selectBlank,
|
||||
boolean selectError
|
||||
) {
|
||||
_rowEvaluable = rowEvaluable;
|
||||
_selectNumeric = selectNumeric;
|
||||
_selectNonNumeric = selectNonNumeric;
|
||||
_selectBlank = selectBlank;
|
||||
_selectError = selectError;
|
||||
}
|
||||
|
||||
public boolean filterRow(Project project, int rowIndex, Row row) {
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
|
||||
Object value = _rowEvaluable.eval(project, rowIndex, row, bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
if (checkValue(v)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
if (checkValue(v)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else if (value instanceof JSONArray) {
|
||||
JSONArray a = (JSONArray) value;
|
||||
int l = a.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
try {
|
||||
if (checkValue(a.get(i))) {
|
||||
return true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
return checkValue(value);
|
||||
}
|
||||
|
||||
protected boolean checkValue(Object v) {
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
return _selectError;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Number) {
|
||||
double d = ((Number) v).doubleValue();
|
||||
if (Double.isInfinite(d) || Double.isNaN(d)) {
|
||||
return _selectError;
|
||||
} else {
|
||||
return _selectNumeric && checkValue(d);
|
||||
}
|
||||
} else {
|
||||
return _selectNonNumeric;
|
||||
}
|
||||
} else {
|
||||
return _selectBlank;
|
||||
}
|
||||
}
|
||||
|
||||
abstract protected boolean checkValue(double d);
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Judge if a row matches by evaluating a given expression on the row, based on a particular
|
||||
* column, and checking the result. It's a match if the result satisfies some string comparisons.
|
||||
*/
|
||||
abstract public class ExpressionStringComparisonRowFilter implements RowFilter {
|
||||
final protected Evaluable _evaluable;
|
||||
final protected String _columnName;
|
||||
final protected int _cellIndex;
|
||||
|
||||
public ExpressionStringComparisonRowFilter(Evaluable evaluable, String columnName, int cellIndex) {
|
||||
_evaluable = evaluable;
|
||||
_columnName = columnName;
|
||||
_cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
public boolean filterRow(Project project, int rowIndex, Row row) {
|
||||
Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
|
||||
|
||||
Object value = _evaluable.evaluate(bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
if (checkValue(v instanceof String ? ((String) v) : v.toString())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
if (checkValue(v.toString())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else if (value instanceof JSONArray) {
|
||||
JSONArray a = (JSONArray) value;
|
||||
int l = a.length();
|
||||
|
||||
for (int i = 0; i < l; i++) {
|
||||
try {
|
||||
if (checkValue(a.get(i).toString())) {
|
||||
return true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
if (checkValue(value instanceof String ? ((String) value) : value.toString())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
abstract protected boolean checkValue(String s);
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
package com.google.gridworks.browsing.filters;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import com.google.gridworks.browsing.util.RowEvaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
|
||||
/**
|
||||
* Judge if a row matches by evaluating a given expression on the row, based on a particular
|
||||
* column, and checking the result. It's a match if the result satisfies some time comparisons,
|
||||
* or if the result is not a time or blank or error and we want non-time or blank or error
|
||||
* values.
|
||||
*/
|
||||
abstract public class ExpressionTimeComparisonRowFilter extends ExpressionNumberComparisonRowFilter {
|
||||
|
||||
final protected boolean _selectTime;
|
||||
final protected boolean _selectNonTime;
|
||||
|
||||
public ExpressionTimeComparisonRowFilter(
|
||||
RowEvaluable rowEvaluable,
|
||||
boolean selectTime,
|
||||
boolean selectNonTime,
|
||||
boolean selectBlank,
|
||||
boolean selectError
|
||||
) {
|
||||
super(rowEvaluable, selectTime, selectNonTime, selectBlank, selectError);
|
||||
_selectTime = selectTime;
|
||||
_selectNonTime = selectNonTime;
|
||||
}
|
||||
|
||||
protected boolean checkValue(Object v) {
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
return _selectError;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Date) {
|
||||
long time = ((Date) v).getTime();
|
||||
return _selectTime && checkValue(time);
|
||||
} else {
|
||||
return _selectNonTime;
|
||||
}
|
||||
} else {
|
||||
return _selectBlank;
|
||||
}
|
||||
}
|
||||
|
||||
// not really needed for operation, just to make extending the abstract class possible
|
||||
protected boolean checkValue(double d) {
|
||||
return false;
|
||||
}
|
||||
|
||||
abstract protected boolean checkValue(long d);
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.RecordFilter;
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
/**
|
||||
* Encapsulate logic for visiting records that match all given record filters.
|
||||
*/
|
||||
public class ConjunctiveFilteredRecords implements FilteredRecords {
|
||||
final protected List<RecordFilter> _recordFilters = new LinkedList<RecordFilter>();
|
||||
|
||||
public void add(RecordFilter recordFilter) {
|
||||
_recordFilters.add(recordFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(Project project, RecordVisitor visitor) {
|
||||
try {
|
||||
visitor.start(project);
|
||||
|
||||
int c = project.recordModel.getRecordCount();
|
||||
for (int r = 0; r < c; r++) {
|
||||
Record record = project.recordModel.getRecord(r);
|
||||
if (matchRecord(project, record)) {
|
||||
if (visitor.visit(project, record)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
visitor.end(project);
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean matchRecord(Project project, Record record) {
|
||||
for (RecordFilter recordFilter : _recordFilters) {
|
||||
if (!recordFilter.filterRecord(project, record)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,51 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RowFilter;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Encapsulate logic for visiting rows that match all give row filters. Also visit
|
||||
* context rows and dependent rows if configured so.
|
||||
*/
|
||||
public class ConjunctiveFilteredRows implements FilteredRows {
|
||||
final protected List<RowFilter> _rowFilters = new LinkedList<RowFilter>();
|
||||
|
||||
public void add(RowFilter rowFilter) {
|
||||
_rowFilters.add(rowFilter);
|
||||
}
|
||||
|
||||
public void accept(Project project, RowVisitor visitor) {
|
||||
try {
|
||||
visitor.start(project);
|
||||
|
||||
int c = project.rows.size();
|
||||
for (int rowIndex = 0; rowIndex < c; rowIndex++) {
|
||||
Row row = project.rows.get(rowIndex);
|
||||
if (matchRow(project, rowIndex, row)) {
|
||||
visitRow(project, visitor, rowIndex, row);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
visitor.end(project);
|
||||
}
|
||||
}
|
||||
|
||||
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row) {
|
||||
visitor.visit(project, rowIndex, row);
|
||||
}
|
||||
|
||||
protected boolean matchRow(Project project, int rowIndex, Row row) {
|
||||
for (RowFilter rowFilter : _rowFilters) {
|
||||
if (!rowFilter.filterRow(project, rowIndex, row)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class ExpressionBasedRowEvaluable implements RowEvaluable {
|
||||
final protected String _columnName;
|
||||
final protected int _cellIndex;
|
||||
final protected Evaluable _eval;
|
||||
|
||||
public ExpressionBasedRowEvaluable(
|
||||
String columnName, int cellIndex, Evaluable eval) {
|
||||
|
||||
_columnName = columnName;
|
||||
_cellIndex = cellIndex;
|
||||
_eval = eval;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object eval(
|
||||
Project project, int rowIndex, Row row, Properties bindings) {
|
||||
|
||||
Cell cell = row.getCell(_cellIndex);
|
||||
|
||||
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
|
||||
|
||||
return _eval.evaluate(bindings);
|
||||
}
|
||||
}
|
||||
@ -1,211 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.browsing.DecoratedValue;
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.browsing.facets.NominalFacetChoice;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Visit matched rows or records and group them into facet choices based on the values computed
|
||||
* from a given expression.
|
||||
*/
|
||||
public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor {
|
||||
static public class IndexedNominalFacetChoice extends NominalFacetChoice {
|
||||
int _latestIndex;
|
||||
|
||||
public IndexedNominalFacetChoice(DecoratedValue decoratedValue, int latestIndex) {
|
||||
super(decoratedValue);
|
||||
_latestIndex = latestIndex;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Configuration
|
||||
*/
|
||||
final protected Evaluable _evaluable;
|
||||
final protected String _columnName;
|
||||
final protected int _cellIndex;
|
||||
|
||||
/*
|
||||
* Computed results
|
||||
*/
|
||||
final public Map<Object, IndexedNominalFacetChoice> choices = new HashMap<Object, IndexedNominalFacetChoice>();
|
||||
public int blankCount = 0;
|
||||
public int errorCount = 0;
|
||||
|
||||
/*
|
||||
* Scratch pad variables
|
||||
*/
|
||||
protected boolean hasBlank;
|
||||
protected boolean hasError;
|
||||
|
||||
public ExpressionNominalValueGrouper(Evaluable evaluable, String columnName, int cellIndex) {
|
||||
_evaluable = evaluable;
|
||||
_columnName = columnName;
|
||||
_cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
hasError = false;
|
||||
hasBlank = false;
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
|
||||
visitRow(project, rowIndex, row, bindings, rowIndex);
|
||||
|
||||
if (hasError) {
|
||||
errorCount++;
|
||||
}
|
||||
if (hasBlank) {
|
||||
blankCount++;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, Record record) {
|
||||
hasError = false;
|
||||
hasBlank = false;
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
Row row = project.rows.get(r);
|
||||
visitRow(project, r, row, bindings, record.recordIndex);
|
||||
}
|
||||
|
||||
if (hasError) {
|
||||
errorCount++;
|
||||
}
|
||||
if (hasBlank) {
|
||||
blankCount++;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void visitRow(Project project, int rowIndex, Row row, Properties bindings, int index) {
|
||||
Object value = evalRow(project, rowIndex, row, bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
processValue(v, rowIndex);
|
||||
}
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
processValue(v, rowIndex);
|
||||
}
|
||||
} else {
|
||||
processValue(value, rowIndex);
|
||||
}
|
||||
} else {
|
||||
processValue(value, rowIndex);
|
||||
}
|
||||
}
|
||||
|
||||
protected Object evalRow(Project project, int rowIndex, Row row, Properties bindings) {
|
||||
Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
|
||||
|
||||
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
|
||||
|
||||
return _evaluable.evaluate(bindings);
|
||||
}
|
||||
|
||||
protected void processValue(Object value, int index) {
|
||||
if (ExpressionUtils.isError(value)) {
|
||||
hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(value)) {
|
||||
String valueString = value.toString();
|
||||
IndexedNominalFacetChoice facetChoice = choices.get(valueString);
|
||||
|
||||
if (facetChoice != null) {
|
||||
if (facetChoice._latestIndex < index) {
|
||||
facetChoice._latestIndex = index;
|
||||
facetChoice.count++;
|
||||
}
|
||||
} else {
|
||||
String label = value.toString();
|
||||
DecoratedValue dValue = new DecoratedValue(value, label);
|
||||
IndexedNominalFacetChoice choice =
|
||||
new IndexedNominalFacetChoice(dValue, index);
|
||||
|
||||
choice.count = 1;
|
||||
choices.put(valueString, choice);
|
||||
}
|
||||
} else {
|
||||
hasBlank = true;
|
||||
}
|
||||
}
|
||||
|
||||
public RowEvaluable getChoiceCountRowEvaluable() {
|
||||
return new RowEvaluable() {
|
||||
@Override
|
||||
public Object eval(Project project, int rowIndex, Row row, Properties bindings) {
|
||||
Object value = evalRow(project, rowIndex, row, bindings);
|
||||
return getChoiceValueCountMultiple(value);
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
public Object getChoiceValueCountMultiple(Object value) {
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] choiceValues = (Object[]) value;
|
||||
List<Integer> counts = new ArrayList<Integer>(choiceValues.length);
|
||||
|
||||
for (int i = 0; i < choiceValues.length; i++) {
|
||||
counts.add(getChoiceValueCount(choiceValues[i]));
|
||||
}
|
||||
return counts;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
List<Object> choiceValues = ExpressionUtils.toObjectList(value);
|
||||
List<Integer> counts = new ArrayList<Integer>(choiceValues.size());
|
||||
|
||||
int count = choiceValues.size();
|
||||
for (int i = 0; i < count; i++) {
|
||||
counts.add(getChoiceValueCount(choiceValues.get(i)));
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
}
|
||||
|
||||
return getChoiceValueCount(value);
|
||||
}
|
||||
|
||||
public Integer getChoiceValueCount(Object choiceValue) {
|
||||
if (ExpressionUtils.isError(choiceValue)) {
|
||||
return errorCount;
|
||||
} else if (ExpressionUtils.isNonBlankData(choiceValue)) {
|
||||
IndexedNominalFacetChoice choice = choices.get(choiceValue);
|
||||
return choice != null ? choice.count : 0;
|
||||
} else {
|
||||
return blankCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,148 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Visit matched rows or records and slot them into bins based on the numbers computed
|
||||
* from a given expression.
|
||||
*/
|
||||
public class ExpressionNumericValueBinner implements RowVisitor, RecordVisitor {
|
||||
/*
|
||||
* Configuration
|
||||
*/
|
||||
final protected RowEvaluable _rowEvaluable;
|
||||
final protected NumericBinIndex _index; // base bins
|
||||
|
||||
/*
|
||||
* Computed results
|
||||
*/
|
||||
final public int[] bins;
|
||||
public int numericCount;
|
||||
public int nonNumericCount;
|
||||
public int blankCount;
|
||||
public int errorCount;
|
||||
|
||||
/*
|
||||
* Scratchpad variables
|
||||
*/
|
||||
protected boolean hasError;
|
||||
protected boolean hasBlank;
|
||||
protected boolean hasNumeric;
|
||||
protected boolean hasNonNumeric;
|
||||
|
||||
public ExpressionNumericValueBinner(RowEvaluable rowEvaluable, NumericBinIndex index) {
|
||||
_rowEvaluable = rowEvaluable;
|
||||
_index = index;
|
||||
bins = new int[_index.getBins().length];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
resetFlags();
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
processRow(project, rowIndex, row, bindings);
|
||||
|
||||
updateCounts();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, Record record) {
|
||||
resetFlags();
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
processRow(project, r, project.rows.get(r), bindings);
|
||||
}
|
||||
|
||||
updateCounts();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void resetFlags() {
|
||||
hasError = false;
|
||||
hasBlank = false;
|
||||
hasNumeric = false;
|
||||
hasNonNumeric = false;
|
||||
}
|
||||
|
||||
protected void updateCounts() {
|
||||
if (hasError) {
|
||||
errorCount++;
|
||||
}
|
||||
if (hasBlank) {
|
||||
blankCount++;
|
||||
}
|
||||
if (hasNumeric) {
|
||||
numericCount++;
|
||||
}
|
||||
if (hasNonNumeric) {
|
||||
nonNumericCount++;
|
||||
}
|
||||
}
|
||||
|
||||
protected void processRow(Project project, int rowIndex, Row row, Properties bindings) {
|
||||
Object value = _rowEvaluable.eval(project, rowIndex, row, bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
processValue(v);
|
||||
}
|
||||
return;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
processValue(v);
|
||||
}
|
||||
return;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
processValue(value);
|
||||
}
|
||||
|
||||
protected void processValue(Object value) {
|
||||
if (ExpressionUtils.isError(value)) {
|
||||
hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(value)) {
|
||||
if (value instanceof Number) {
|
||||
double d = ((Number) value).doubleValue();
|
||||
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
|
||||
hasNumeric = true;
|
||||
|
||||
int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep());
|
||||
if (bin >= 0 && bin < bins.length) { // as a precaution
|
||||
bins[bin]++;
|
||||
}
|
||||
} else {
|
||||
hasError = true;
|
||||
}
|
||||
} else {
|
||||
hasNonNumeric = true;
|
||||
}
|
||||
} else {
|
||||
hasBlank = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,146 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* Visit matched rows or records and slot them into bins based on the date computed
|
||||
* from a given expression.
|
||||
*/
|
||||
public class ExpressionTimeValueBinner implements RowVisitor, RecordVisitor {
|
||||
|
||||
/*
|
||||
* Configuration
|
||||
*/
|
||||
final protected RowEvaluable _rowEvaluable;
|
||||
final protected TimeBinIndex _index; // base bins
|
||||
|
||||
/*
|
||||
* Computed results
|
||||
*/
|
||||
final public int[] bins;
|
||||
public int timeCount;
|
||||
public int nonTimeCount;
|
||||
public int blankCount;
|
||||
public int errorCount;
|
||||
|
||||
/*
|
||||
* Scratchpad variables
|
||||
*/
|
||||
protected boolean hasError;
|
||||
protected boolean hasBlank;
|
||||
protected boolean hasTime;
|
||||
protected boolean hasNonTime;
|
||||
|
||||
public ExpressionTimeValueBinner(RowEvaluable rowEvaluable, TimeBinIndex index) {
|
||||
_rowEvaluable = rowEvaluable;
|
||||
_index = index;
|
||||
bins = new int[_index.getBins().length];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
resetFlags();
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
processRow(project, rowIndex, row, bindings);
|
||||
|
||||
updateCounts();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, Record record) {
|
||||
resetFlags();
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
processRow(project, r, project.rows.get(r), bindings);
|
||||
}
|
||||
|
||||
updateCounts();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void resetFlags() {
|
||||
hasError = false;
|
||||
hasBlank = false;
|
||||
hasTime = false;
|
||||
hasNonTime = false;
|
||||
}
|
||||
|
||||
protected void updateCounts() {
|
||||
if (hasError) {
|
||||
errorCount++;
|
||||
}
|
||||
if (hasBlank) {
|
||||
blankCount++;
|
||||
}
|
||||
if (hasTime) {
|
||||
timeCount++;
|
||||
}
|
||||
if (hasNonTime) {
|
||||
nonTimeCount++;
|
||||
}
|
||||
}
|
||||
|
||||
protected void processRow(Project project, int rowIndex, Row row, Properties bindings) {
|
||||
Object value = _rowEvaluable.eval(project, rowIndex, row, bindings);
|
||||
if (value != null) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
processValue(v);
|
||||
}
|
||||
return;
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
processValue(v);
|
||||
}
|
||||
return;
|
||||
} // else, fall through
|
||||
}
|
||||
|
||||
processValue(value);
|
||||
}
|
||||
|
||||
protected void processValue(Object value) {
|
||||
if (ExpressionUtils.isError(value)) {
|
||||
hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(value)) {
|
||||
if (value instanceof Date) {
|
||||
long t = ((Date) value).getTime();
|
||||
hasTime = true;
|
||||
|
||||
int bin = (int) Math.floor((t - _index.getMin()) / _index.getStep());
|
||||
if (bin >= 0 && bin < bins.length) { // as a precaution
|
||||
bins[bin]++;
|
||||
}
|
||||
} else {
|
||||
hasNonTime = true;
|
||||
}
|
||||
} else {
|
||||
hasBlank = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import com.google.gridworks.browsing.FilteredRecords;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class FilteredRecordsAsFilteredRows implements FilteredRows {
|
||||
final protected FilteredRecords _filteredRecords;
|
||||
|
||||
public FilteredRecordsAsFilteredRows(FilteredRecords filteredRecords) {
|
||||
_filteredRecords = filteredRecords;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(Project project, RowVisitor visitor) {
|
||||
_filteredRecords.accept(project, new RowVisitorAsRecordVisitor(visitor));
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,226 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* A utility class for computing the base bins that form the base histograms of
|
||||
* numeric range facets. It evaluates an expression on all the rows of a project to
|
||||
* get numeric values, determines how many bins to distribute those values in, and
|
||||
* bins the rows accordingly.
|
||||
*
|
||||
* This class processes all rows rather than just the filtered rows because it
|
||||
* needs to compute the base bins of a numeric range facet, which remain unchanged
|
||||
* as the user interacts with the facet.
|
||||
*/
|
||||
abstract public class NumericBinIndex {
|
||||
|
||||
protected int _totalValueCount;
|
||||
protected int _numbericValueCount;
|
||||
protected double _min;
|
||||
protected double _max;
|
||||
protected double _step;
|
||||
protected int[] _bins;
|
||||
|
||||
protected int _numericRowCount;
|
||||
protected int _nonNumericRowCount;
|
||||
protected int _blankRowCount;
|
||||
protected int _errorRowCount;
|
||||
|
||||
protected boolean _hasError = false;
|
||||
protected boolean _hasNonNumeric = false;
|
||||
protected boolean _hasNumeric = false;
|
||||
protected boolean _hasBlank = false;
|
||||
|
||||
abstract protected void iterate(Project project, RowEvaluable rowEvaluable, List<Double> allValues);
|
||||
|
||||
public NumericBinIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
_min = Double.POSITIVE_INFINITY;
|
||||
_max = Double.NEGATIVE_INFINITY;
|
||||
|
||||
List<Double> allValues = new ArrayList<Double>();
|
||||
|
||||
iterate(project, rowEvaluable, allValues);
|
||||
|
||||
_numbericValueCount = allValues.size();
|
||||
|
||||
if (_min >= _max) {
|
||||
_step = 1;
|
||||
_min = Math.min(_min, _max);
|
||||
_max = _step;
|
||||
_bins = new int[1];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
double diff = _max - _min;
|
||||
|
||||
_step = 1;
|
||||
if (diff > 10) {
|
||||
while (_step * 100 < diff) {
|
||||
_step *= 10;
|
||||
}
|
||||
} else {
|
||||
while (_step * 100 > diff) {
|
||||
_step /= 10;
|
||||
}
|
||||
}
|
||||
|
||||
double originalMax = _max;
|
||||
_min = (Math.floor(_min / _step) * _step);
|
||||
_max = (Math.ceil(_max / _step) * _step);
|
||||
|
||||
double binCount = (_max - _min) / _step;
|
||||
if (binCount > 100) {
|
||||
_step *= 2;
|
||||
binCount = (binCount + 1) / 2;
|
||||
}
|
||||
|
||||
if (_max <= originalMax) {
|
||||
_max += _step;
|
||||
binCount++;
|
||||
}
|
||||
|
||||
_bins = new int[(int) Math.round(binCount)];
|
||||
for (double d : allValues) {
|
||||
int bin = Math.max((int) Math.floor((d - _min) / _step),0);
|
||||
_bins[bin]++;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isNumeric() {
|
||||
return _numbericValueCount > _totalValueCount / 2;
|
||||
}
|
||||
|
||||
public double getMin() {
|
||||
return _min;
|
||||
}
|
||||
|
||||
public double getMax() {
|
||||
return _max;
|
||||
}
|
||||
|
||||
public double getStep() {
|
||||
return _step;
|
||||
}
|
||||
|
||||
public int[] getBins() {
|
||||
return _bins;
|
||||
}
|
||||
|
||||
public int getNumericRowCount() {
|
||||
return _numericRowCount;
|
||||
}
|
||||
|
||||
public int getNonNumericRowCount() {
|
||||
return _nonNumericRowCount;
|
||||
}
|
||||
|
||||
public int getBlankRowCount() {
|
||||
return _blankRowCount;
|
||||
}
|
||||
|
||||
public int getErrorRowCount() {
|
||||
return _errorRowCount;
|
||||
}
|
||||
|
||||
protected void processRow(
|
||||
Project project,
|
||||
RowEvaluable rowEvaluable,
|
||||
List<Double> allValues,
|
||||
int rowIndex,
|
||||
Row row,
|
||||
Properties bindings
|
||||
) {
|
||||
Object value = rowEvaluable.eval(project, rowIndex, row, bindings);
|
||||
|
||||
if (ExpressionUtils.isError(value)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(value)) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
_totalValueCount++;
|
||||
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Number) {
|
||||
_hasNumeric = true;
|
||||
processValue(((Number) v).doubleValue(), allValues);
|
||||
} else {
|
||||
_hasNonNumeric = true;
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
_totalValueCount++;
|
||||
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Number) {
|
||||
_hasNumeric = true;
|
||||
processValue(((Number) v).doubleValue(), allValues);
|
||||
} else {
|
||||
_hasNonNumeric = true;
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_totalValueCount++;
|
||||
|
||||
if (value instanceof Number) {
|
||||
_hasNumeric = true;
|
||||
processValue(((Number) value).doubleValue(), allValues);
|
||||
} else {
|
||||
_hasNonNumeric = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
|
||||
protected void preprocessing() {
|
||||
_hasBlank = false;
|
||||
_hasError = false;
|
||||
_hasNonNumeric = false;
|
||||
_hasNumeric = false;
|
||||
}
|
||||
|
||||
protected void postprocessing() {
|
||||
if (_hasError) {
|
||||
_errorRowCount++;
|
||||
}
|
||||
if (_hasBlank) {
|
||||
_blankRowCount++;
|
||||
}
|
||||
if (_hasNumeric) {
|
||||
_numericRowCount++;
|
||||
}
|
||||
if (_hasNonNumeric) {
|
||||
_nonNumericRowCount++;
|
||||
}
|
||||
}
|
||||
|
||||
protected void processValue(double v, List<Double> allValues) {
|
||||
if (!Double.isInfinite(v) && !Double.isNaN(v)) {
|
||||
_min = Math.min(_min, v);
|
||||
_max = Math.max(_max, v);
|
||||
allValues.add(v);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class NumericBinRecordIndex extends NumericBinIndex {
|
||||
public NumericBinRecordIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
super(project, rowEvaluable);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void iterate(
|
||||
Project project, RowEvaluable rowEvaluable, List<Double> allValues) {
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
int count = project.recordModel.getRecordCount();
|
||||
|
||||
for (int r = 0; r < count; r++) {
|
||||
Record record = project.recordModel.getRecord(r);
|
||||
|
||||
preprocessing();
|
||||
|
||||
for (int i = record.fromRowIndex; i < record.toRowIndex; i++) {
|
||||
Row row = project.rows.get(i);
|
||||
|
||||
processRow(project, rowEvaluable, allValues, i, row, bindings);
|
||||
}
|
||||
|
||||
postprocessing();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class NumericBinRowIndex extends NumericBinIndex {
|
||||
public NumericBinRowIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
|
||||
super(project, rowEvaluable);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void iterate(
|
||||
Project project, RowEvaluable rowEvaluable, List<Double> allValues) {
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
|
||||
for (int i = 0; i < project.rows.size(); i++) {
|
||||
Row row = project.rows.get(i);
|
||||
|
||||
preprocessing();
|
||||
|
||||
processRow(project, rowEvaluable, allValues, i, row, bindings);
|
||||
|
||||
postprocessing();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public interface RowEvaluable {
|
||||
public Object eval(Project project, int rowIndex, Row row, Properties bindings);
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import com.google.gridworks.browsing.RecordVisitor;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
|
||||
public class RowVisitorAsRecordVisitor implements RecordVisitor {
|
||||
final protected RowVisitor _rowVisitor;
|
||||
|
||||
public RowVisitorAsRecordVisitor(RowVisitor rowVisitor) {
|
||||
_rowVisitor = rowVisitor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
_rowVisitor.start(project);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
_rowVisitor.end(project);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, Record record) {
|
||||
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
|
||||
if (_rowVisitor.visit(project, r, project.rows.get(r))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -1,218 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
/**
|
||||
* A utility class for computing the base bins that form the base histograms of
|
||||
* temporal range facets. It evaluates an expression on all the rows of a project to
|
||||
* get temporal values, determines how many bins to distribute those values in, and
|
||||
* bins the rows accordingly.
|
||||
*
|
||||
* This class processes all rows rather than just the filtered rows because it
|
||||
* needs to compute the base bins of a temporal range facet, which remain unchanged
|
||||
* as the user interacts with the facet.
|
||||
*/
|
||||
abstract public class TimeBinIndex {
|
||||
|
||||
protected int _totalValueCount;
|
||||
protected int _timeValueCount;
|
||||
protected long _min;
|
||||
protected long _max;
|
||||
protected long _step;
|
||||
protected int[] _bins;
|
||||
|
||||
protected int _timeRowCount;
|
||||
protected int _nonTimeRowCount;
|
||||
protected int _blankRowCount;
|
||||
protected int _errorRowCount;
|
||||
|
||||
protected boolean _hasError = false;
|
||||
protected boolean _hasNonTime = false;
|
||||
protected boolean _hasTime = false;
|
||||
protected boolean _hasBlank = false;
|
||||
|
||||
protected long[] steps = {
|
||||
1, // msec
|
||||
1000, // sec
|
||||
1000*60, // min
|
||||
1000*60*60, // hour
|
||||
1000*60*60*24, // day
|
||||
1000*60*60*24*7, // week
|
||||
1000*2629746, // month (average Gregorian year / 12)
|
||||
1000*31556952, // year (average Gregorian year)
|
||||
1000*31556952*10, // decade
|
||||
1000*31556952*100, // century
|
||||
1000*31556952*1000, // millennium
|
||||
};
|
||||
|
||||
abstract protected void iterate(Project project, RowEvaluable rowEvaluable, List<Long> allValues);
|
||||
|
||||
public TimeBinIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
_min = Long.MAX_VALUE;
|
||||
_max = Long.MIN_VALUE;
|
||||
|
||||
List<Long> allValues = new ArrayList<Long>();
|
||||
|
||||
iterate(project, rowEvaluable, allValues);
|
||||
|
||||
_timeValueCount = allValues.size();
|
||||
|
||||
if (_min >= _max) {
|
||||
_step = 1;
|
||||
_min = Math.min(_min, _max);
|
||||
_max = _step;
|
||||
_bins = new int[1];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
long diff = _max - _min;
|
||||
|
||||
for (int i = 0; i < steps.length; i++) {
|
||||
_step = steps[i];
|
||||
if (diff / _step <= 100) break;
|
||||
}
|
||||
|
||||
_bins = new int[(int) (diff / _step) + 1];
|
||||
for (long d : allValues) {
|
||||
int bin = (int) Math.max((d - _min) / _step,0);
|
||||
_bins[bin]++;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isTemporal() {
|
||||
return _timeValueCount > _totalValueCount / 2;
|
||||
}
|
||||
|
||||
public long getMin() {
|
||||
return _min;
|
||||
}
|
||||
|
||||
public long getMax() {
|
||||
return _max;
|
||||
}
|
||||
|
||||
public long getStep() {
|
||||
return _step;
|
||||
}
|
||||
|
||||
public int[] getBins() {
|
||||
return _bins;
|
||||
}
|
||||
|
||||
public int getTimeRowCount() {
|
||||
return _timeRowCount;
|
||||
}
|
||||
|
||||
public int getNonTimeRowCount() {
|
||||
return _nonTimeRowCount;
|
||||
}
|
||||
|
||||
public int getBlankRowCount() {
|
||||
return _blankRowCount;
|
||||
}
|
||||
|
||||
public int getErrorRowCount() {
|
||||
return _errorRowCount;
|
||||
}
|
||||
|
||||
protected void processRow(
|
||||
Project project,
|
||||
RowEvaluable rowEvaluable,
|
||||
List<Long> allValues,
|
||||
int rowIndex,
|
||||
Row row,
|
||||
Properties bindings
|
||||
) {
|
||||
Object value = rowEvaluable.eval(project, rowIndex, row, bindings);
|
||||
|
||||
if (ExpressionUtils.isError(value)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(value)) {
|
||||
if (value.getClass().isArray()) {
|
||||
Object[] a = (Object[]) value;
|
||||
for (Object v : a) {
|
||||
_totalValueCount++;
|
||||
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Date) {
|
||||
_hasTime = true;
|
||||
processValue(((Date) v).getTime(), allValues);
|
||||
} else {
|
||||
_hasNonTime = true;
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Collection<?>) {
|
||||
for (Object v : ExpressionUtils.toObjectCollection(value)) {
|
||||
_totalValueCount++;
|
||||
|
||||
if (ExpressionUtils.isError(v)) {
|
||||
_hasError = true;
|
||||
} else if (ExpressionUtils.isNonBlankData(v)) {
|
||||
if (v instanceof Date) {
|
||||
_hasTime = true;
|
||||
processValue(((Date) v).getTime(), allValues);
|
||||
} else {
|
||||
_hasNonTime = true;
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_totalValueCount++;
|
||||
|
||||
if (value instanceof Date) {
|
||||
_hasTime = true;
|
||||
processValue(((Date) value).getTime(), allValues);
|
||||
} else {
|
||||
_hasNonTime = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_hasBlank = true;
|
||||
}
|
||||
}
|
||||
|
||||
protected void preprocessing() {
|
||||
_hasBlank = false;
|
||||
_hasError = false;
|
||||
_hasNonTime = false;
|
||||
_hasTime = false;
|
||||
}
|
||||
|
||||
protected void postprocessing() {
|
||||
if (_hasError) {
|
||||
_errorRowCount++;
|
||||
}
|
||||
if (_hasBlank) {
|
||||
_blankRowCount++;
|
||||
}
|
||||
if (_hasTime) {
|
||||
_timeRowCount++;
|
||||
}
|
||||
if (_hasNonTime) {
|
||||
_nonTimeRowCount++;
|
||||
}
|
||||
}
|
||||
|
||||
protected void processValue(long v, List<Long> allValues) {
|
||||
_min = Math.min(_min, v);
|
||||
_max = Math.max(_max, v);
|
||||
allValues.add(v);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Record;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class TimeBinRecordIndex extends TimeBinIndex {
|
||||
|
||||
public TimeBinRecordIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
super(project, rowEvaluable);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void iterate(Project project, RowEvaluable rowEvaluable, List<Long> allValues) {
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
int count = project.recordModel.getRecordCount();
|
||||
|
||||
for (int r = 0; r < count; r++) {
|
||||
Record record = project.recordModel.getRecord(r);
|
||||
|
||||
preprocessing();
|
||||
|
||||
for (int i = record.fromRowIndex; i < record.toRowIndex; i++) {
|
||||
Row row = project.rows.get(i);
|
||||
|
||||
processRow(project, rowEvaluable, allValues, i, row, bindings);
|
||||
}
|
||||
|
||||
postprocessing();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
package com.google.gridworks.browsing.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.gridworks.expr.ExpressionUtils;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class TimeBinRowIndex extends TimeBinIndex {
|
||||
|
||||
public TimeBinRowIndex(Project project, RowEvaluable rowEvaluable) {
|
||||
super(project, rowEvaluable);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void iterate(Project project, RowEvaluable rowEvaluable, List<Long> allValues) {
|
||||
|
||||
Properties bindings = ExpressionUtils.createBindings(project);
|
||||
|
||||
for (int i = 0; i < project.rows.size(); i++) {
|
||||
Row row = project.rows.get(i);
|
||||
|
||||
preprocessing();
|
||||
|
||||
processRow(project, rowEvaluable, allValues, i, row, bindings);
|
||||
|
||||
postprocessing();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
package com.google.gridworks.clustering;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.Jsonizable;
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public abstract class Clusterer implements Jsonizable {
|
||||
|
||||
protected Project _project;
|
||||
protected int _colindex;
|
||||
protected JSONObject _config;
|
||||
|
||||
public abstract void computeClusters(Engine engine);
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
_project = project;
|
||||
_config = o;
|
||||
|
||||
String colname = o.getString("column");
|
||||
for (Column column : project.columnModel.columns) {
|
||||
if (column.getName().equals(colname)) {
|
||||
_colindex = column.getCellIndex();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,169 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.TreeMap;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.clustering.Clusterer;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
public class BinningClusterer extends Clusterer {
|
||||
|
||||
private Keyer _keyer;
|
||||
|
||||
static final protected Map<String, Keyer> _keyers = new HashMap<String, Keyer>();
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("binning_clusterer");
|
||||
|
||||
List<Map<String,Integer>> _clusters;
|
||||
|
||||
static {
|
||||
_keyers.put("fingerprint", new FingerprintKeyer());
|
||||
_keyers.put("ngram-fingerprint", new NGramFingerprintKeyer());
|
||||
_keyers.put("metaphone", new MetaphoneKeyer());
|
||||
_keyers.put("double-metaphone", new DoubleMetaphoneKeyer());
|
||||
_keyers.put("soundex", new SoundexKeyer());
|
||||
}
|
||||
|
||||
class BinningRowVisitor implements RowVisitor {
|
||||
|
||||
Keyer _keyer;
|
||||
Object[] _params;
|
||||
JSONObject _config;
|
||||
|
||||
Map<String,Map<String,Integer>> _map = new HashMap<String,Map<String,Integer>>();
|
||||
|
||||
public BinningRowVisitor(Keyer k, JSONObject o) {
|
||||
_keyer = k;
|
||||
_config = o;
|
||||
if (k instanceof NGramFingerprintKeyer) {
|
||||
try {
|
||||
int size = _config.getJSONObject("params").getInt("ngram-size");
|
||||
logger.debug("Using ngram size: {}", size);
|
||||
_params = new Object[1];
|
||||
_params[0] = size;
|
||||
} catch (JSONException e) {
|
||||
//Gridworks.warn("No params specified, using default");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
Cell cell = row.getCell(_colindex);
|
||||
if (cell != null && cell.value != null) {
|
||||
Object v = cell.value;
|
||||
String s = (v instanceof String) ? ((String) v) : v.toString();
|
||||
String key = _keyer.key(s,_params);
|
||||
if (_map.containsKey(key)) {
|
||||
Map<String,Integer> m = _map.get(key);
|
||||
if (m.containsKey(s)) {
|
||||
m.put(s, m.get(s) + 1);
|
||||
} else {
|
||||
m.put(s,1);
|
||||
}
|
||||
} else {
|
||||
Map<String,Integer> m = new TreeMap<String,Integer>();
|
||||
m.put(s,1);
|
||||
_map.put(key, m);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Map<String,Map<String,Integer>> getMap() {
|
||||
return _map;
|
||||
}
|
||||
}
|
||||
|
||||
public static class SizeComparator implements Comparator<Map<String,Integer>>, Serializable {
|
||||
private static final long serialVersionUID = -1390696157208674054L;
|
||||
public int compare(Map<String,Integer> o1, Map<String,Integer> o2) {
|
||||
int s1 = o1.size();
|
||||
int s2 = o2.size();
|
||||
if (o1 == o2) {
|
||||
int total1 = 0;
|
||||
for (int i : o1.values()) {
|
||||
total1 += i;
|
||||
}
|
||||
int total2 = 0;
|
||||
for (int i : o2.values()) {
|
||||
total2 += i;
|
||||
}
|
||||
return total2 - total1;
|
||||
} else {
|
||||
return s2 - s1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class EntriesComparator implements Comparator<Entry<String,Integer>>, Serializable {
|
||||
private static final long serialVersionUID = 2763378036791777964L;
|
||||
public int compare(Entry<String,Integer> o1, Entry<String,Integer> o2) {
|
||||
return o2.getValue() - o1.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
super.initializeFromJSON(project, o);
|
||||
_keyer = _keyers.get(o.getString("function").toLowerCase());
|
||||
}
|
||||
|
||||
public void computeClusters(Engine engine) {
|
||||
BinningRowVisitor visitor = new BinningRowVisitor(_keyer,_config);
|
||||
FilteredRows filteredRows = engine.getAllFilteredRows();
|
||||
filteredRows.accept(_project, visitor);
|
||||
|
||||
Map<String,Map<String,Integer>> map = visitor.getMap();
|
||||
_clusters = new ArrayList<Map<String,Integer>>(map.values());
|
||||
Collections.sort(_clusters, new SizeComparator());
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
EntriesComparator c = new EntriesComparator();
|
||||
|
||||
writer.array();
|
||||
for (Map<String,Integer> m : _clusters) {
|
||||
if (m.size() > 1) {
|
||||
writer.array();
|
||||
List<Entry<String,Integer>> entries = new ArrayList<Entry<String,Integer>>(m.entrySet());
|
||||
Collections.sort(entries,c);
|
||||
for (Entry<String,Integer> e : entries) {
|
||||
writer.object();
|
||||
writer.key("v"); writer.value(e.getKey());
|
||||
writer.key("c"); writer.value(e.getValue());
|
||||
writer.endObject();
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import org.apache.commons.codec.language.DoubleMetaphone;
|
||||
|
||||
public class DoubleMetaphoneKeyer extends Keyer {
|
||||
|
||||
private DoubleMetaphone _metaphone2;
|
||||
|
||||
public DoubleMetaphoneKeyer() {
|
||||
_metaphone2 = new DoubleMetaphone();
|
||||
_metaphone2.setMaxCodeLen(2000);
|
||||
}
|
||||
|
||||
public String key(String s, Object... o) {
|
||||
return _metaphone2.doubleMetaphone(s);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,249 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
public class FingerprintKeyer extends Keyer {
|
||||
|
||||
static final Pattern alphanum = Pattern.compile("\\p{Punct}|\\p{Cntrl}");
|
||||
|
||||
public String key(String s, Object... o) {
|
||||
s = s.trim(); // first off, remove whitespace around the string
|
||||
s = s.toLowerCase(); // then lowercase it
|
||||
s = alphanum.matcher(s).replaceAll(""); // then remove all punctuation and control chars
|
||||
String[] frags = StringUtils.split(s); // split by whitespace
|
||||
TreeSet<String> set = new TreeSet<String>();
|
||||
for (String ss : frags) {
|
||||
set.add(ss); // order fragments and dedupe
|
||||
}
|
||||
StringBuffer b = new StringBuffer();
|
||||
Iterator<String> i = set.iterator();
|
||||
while (i.hasNext()) { // join ordered fragments back together
|
||||
b.append(i.next());
|
||||
b.append(' ');
|
||||
}
|
||||
return asciify(b.toString()); // find ASCII equivalent to characters
|
||||
}
|
||||
|
||||
protected String asciify(String s) {
|
||||
char[] c = s.toCharArray();
|
||||
StringBuffer b = new StringBuffer();
|
||||
for (int i = 0; i < c.length; i++) {
|
||||
b.append(translate(c[i]));
|
||||
}
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate the given unicode char in the closest ASCII representation
|
||||
* NOTE: this function deals only with latin-1 supplement and latin-1 extended code charts
|
||||
*/
|
||||
private char translate(char c) {
|
||||
switch(c) {
|
||||
case '\u00C0':
|
||||
case '\u00C1':
|
||||
case '\u00C2':
|
||||
case '\u00C3':
|
||||
case '\u00C4':
|
||||
case '\u00C5':
|
||||
case '\u00E0':
|
||||
case '\u00E1':
|
||||
case '\u00E2':
|
||||
case '\u00E3':
|
||||
case '\u00E4':
|
||||
case '\u00E5':
|
||||
case '\u0100':
|
||||
case '\u0101':
|
||||
case '\u0102':
|
||||
case '\u0103':
|
||||
case '\u0104':
|
||||
case '\u0105':
|
||||
return 'a';
|
||||
case '\u00C7':
|
||||
case '\u00E7':
|
||||
case '\u0106':
|
||||
case '\u0107':
|
||||
case '\u0108':
|
||||
case '\u0109':
|
||||
case '\u010A':
|
||||
case '\u010B':
|
||||
case '\u010C':
|
||||
case '\u010D':
|
||||
return 'c';
|
||||
case '\u00D0':
|
||||
case '\u00F0':
|
||||
case '\u010E':
|
||||
case '\u010F':
|
||||
case '\u0110':
|
||||
case '\u0111':
|
||||
return 'd';
|
||||
case '\u00C8':
|
||||
case '\u00C9':
|
||||
case '\u00CA':
|
||||
case '\u00CB':
|
||||
case '\u00E8':
|
||||
case '\u00E9':
|
||||
case '\u00EA':
|
||||
case '\u00EB':
|
||||
case '\u0112':
|
||||
case '\u0113':
|
||||
case '\u0114':
|
||||
case '\u0115':
|
||||
case '\u0116':
|
||||
case '\u0117':
|
||||
case '\u0118':
|
||||
case '\u0119':
|
||||
case '\u011A':
|
||||
case '\u011B':
|
||||
return 'e';
|
||||
case '\u011C':
|
||||
case '\u011D':
|
||||
case '\u011E':
|
||||
case '\u011F':
|
||||
case '\u0120':
|
||||
case '\u0121':
|
||||
case '\u0122':
|
||||
case '\u0123':
|
||||
return 'g';
|
||||
case '\u0124':
|
||||
case '\u0125':
|
||||
case '\u0126':
|
||||
case '\u0127':
|
||||
return 'h';
|
||||
case '\u00CC':
|
||||
case '\u00CD':
|
||||
case '\u00CE':
|
||||
case '\u00CF':
|
||||
case '\u00EC':
|
||||
case '\u00ED':
|
||||
case '\u00EE':
|
||||
case '\u00EF':
|
||||
case '\u0128':
|
||||
case '\u0129':
|
||||
case '\u012A':
|
||||
case '\u012B':
|
||||
case '\u012C':
|
||||
case '\u012D':
|
||||
case '\u012E':
|
||||
case '\u012F':
|
||||
case '\u0130':
|
||||
case '\u0131':
|
||||
return 'i';
|
||||
case '\u0134':
|
||||
case '\u0135':
|
||||
return 'j';
|
||||
case '\u0136':
|
||||
case '\u0137':
|
||||
case '\u0138':
|
||||
return 'k';
|
||||
case '\u0139':
|
||||
case '\u013A':
|
||||
case '\u013B':
|
||||
case '\u013C':
|
||||
case '\u013D':
|
||||
case '\u013E':
|
||||
case '\u013F':
|
||||
case '\u0140':
|
||||
case '\u0141':
|
||||
case '\u0142':
|
||||
return 'l';
|
||||
case '\u00D1':
|
||||
case '\u00F1':
|
||||
case '\u0143':
|
||||
case '\u0144':
|
||||
case '\u0145':
|
||||
case '\u0146':
|
||||
case '\u0147':
|
||||
case '\u0148':
|
||||
case '\u0149':
|
||||
case '\u014A':
|
||||
case '\u014B':
|
||||
return 'n';
|
||||
case '\u00D2':
|
||||
case '\u00D3':
|
||||
case '\u00D4':
|
||||
case '\u00D5':
|
||||
case '\u00D6':
|
||||
case '\u00D8':
|
||||
case '\u00F2':
|
||||
case '\u00F3':
|
||||
case '\u00F4':
|
||||
case '\u00F5':
|
||||
case '\u00F6':
|
||||
case '\u00F8':
|
||||
case '\u014C':
|
||||
case '\u014D':
|
||||
case '\u014E':
|
||||
case '\u014F':
|
||||
case '\u0150':
|
||||
case '\u0151':
|
||||
return 'o';
|
||||
case '\u0154':
|
||||
case '\u0155':
|
||||
case '\u0156':
|
||||
case '\u0157':
|
||||
case '\u0158':
|
||||
case '\u0159':
|
||||
return 'r';
|
||||
case '\u015A':
|
||||
case '\u015B':
|
||||
case '\u015C':
|
||||
case '\u015D':
|
||||
case '\u015E':
|
||||
case '\u015F':
|
||||
case '\u0160':
|
||||
case '\u0161':
|
||||
case '\u017F':
|
||||
return 's';
|
||||
case '\u0162':
|
||||
case '\u0163':
|
||||
case '\u0164':
|
||||
case '\u0165':
|
||||
case '\u0166':
|
||||
case '\u0167':
|
||||
return 't';
|
||||
case '\u00D9':
|
||||
case '\u00DA':
|
||||
case '\u00DB':
|
||||
case '\u00DC':
|
||||
case '\u00F9':
|
||||
case '\u00FA':
|
||||
case '\u00FB':
|
||||
case '\u00FC':
|
||||
case '\u0168':
|
||||
case '\u0169':
|
||||
case '\u016A':
|
||||
case '\u016B':
|
||||
case '\u016C':
|
||||
case '\u016D':
|
||||
case '\u016E':
|
||||
case '\u016F':
|
||||
case '\u0170':
|
||||
case '\u0171':
|
||||
case '\u0172':
|
||||
case '\u0173':
|
||||
return 'u';
|
||||
case '\u0174':
|
||||
case '\u0175':
|
||||
return 'w';
|
||||
case '\u00DD':
|
||||
case '\u00FD':
|
||||
case '\u00FF':
|
||||
case '\u0176':
|
||||
case '\u0177':
|
||||
case '\u0178':
|
||||
return 'y';
|
||||
case '\u0179':
|
||||
case '\u017A':
|
||||
case '\u017B':
|
||||
case '\u017C':
|
||||
case '\u017D':
|
||||
case '\u017E':
|
||||
return 'z';
|
||||
}
|
||||
return c;
|
||||
}
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
|
||||
public abstract class Keyer {
|
||||
|
||||
public String key(String s) {
|
||||
return this.key(s, (Object[]) null);
|
||||
}
|
||||
|
||||
public abstract String key(String string, Object... params);
|
||||
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import org.apache.commons.codec.language.Metaphone;
|
||||
|
||||
public class MetaphoneKeyer extends Keyer {
|
||||
|
||||
private Metaphone _metaphone;
|
||||
|
||||
public MetaphoneKeyer() {
|
||||
_metaphone = new Metaphone();
|
||||
_metaphone.setMaxCodeLen(2000);
|
||||
}
|
||||
|
||||
public String key(String s, Object... o) {
|
||||
return _metaphone.metaphone(s);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class NGramFingerprintKeyer extends FingerprintKeyer {
|
||||
|
||||
static final Pattern alphanum = Pattern.compile("\\p{Punct}|\\p{Cntrl}|\\p{Space}");
|
||||
|
||||
public String key(String s, Object... o) {
|
||||
int ngram_size = 2;
|
||||
if (o != null && o.length > 0 && o[0] instanceof Number) {
|
||||
ngram_size = (Integer) o[0];
|
||||
}
|
||||
s = s.toLowerCase(); // then lowercase it
|
||||
s = alphanum.matcher(s).replaceAll(""); // then remove all punctuation and control chars
|
||||
TreeSet<String> set = ngram_split(s,ngram_size);
|
||||
StringBuffer b = new StringBuffer();
|
||||
Iterator<String> i = set.iterator();
|
||||
while (i.hasNext()) { // join ordered fragments back together
|
||||
b.append(i.next());
|
||||
}
|
||||
return asciify(b.toString()); // find ASCII equivalent to characters
|
||||
}
|
||||
|
||||
protected TreeSet<String> ngram_split(String s, int size) {
|
||||
TreeSet<String> set = new TreeSet<String>();
|
||||
char[] chars = s.toCharArray();
|
||||
for (int i = 0; i + size <= chars.length; i++) {
|
||||
set.add(new String(chars,i,size));
|
||||
}
|
||||
return set;
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
package com.google.gridworks.clustering.binning;
|
||||
|
||||
import org.apache.commons.codec.language.Soundex;
|
||||
|
||||
public class SoundexKeyer extends Keyer {
|
||||
|
||||
private Soundex _soundex;
|
||||
|
||||
public SoundexKeyer() {
|
||||
_soundex = new Soundex();
|
||||
}
|
||||
|
||||
public String key(String s, Object... o) {
|
||||
return _soundex.soundex(s);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,211 +0,0 @@
|
||||
package com.google.gridworks.clustering.knn;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.RowVisitor;
|
||||
import com.google.gridworks.clustering.Clusterer;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.Row;
|
||||
|
||||
import edu.mit.simile.vicino.clustering.NGramClusterer;
|
||||
import edu.mit.simile.vicino.clustering.VPTreeClusterer;
|
||||
import edu.mit.simile.vicino.distances.BZip2Distance;
|
||||
import edu.mit.simile.vicino.distances.Distance;
|
||||
import edu.mit.simile.vicino.distances.GZipDistance;
|
||||
import edu.mit.simile.vicino.distances.JaccardDistance;
|
||||
import edu.mit.simile.vicino.distances.JaroDistance;
|
||||
import edu.mit.simile.vicino.distances.JaroWinklerDistance;
|
||||
import edu.mit.simile.vicino.distances.JaroWinklerTFIDFDistance;
|
||||
import edu.mit.simile.vicino.distances.LevenshteinDistance;
|
||||
import edu.mit.simile.vicino.distances.PPMDistance;
|
||||
|
||||
public class kNNClusterer extends Clusterer {
|
||||
|
||||
private Distance _distance;
|
||||
|
||||
static final protected Map<String, Distance> _distances = new HashMap<String, Distance>();
|
||||
|
||||
List<Set<Serializable>> _clusters;
|
||||
|
||||
Map<Serializable, Integer> _counts = new HashMap<Serializable, Integer>();
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("kNN_clusterer");
|
||||
|
||||
static {
|
||||
_distances.put("levenshtein", new LevenshteinDistance());
|
||||
_distances.put("jaccard", new JaccardDistance());
|
||||
_distances.put("jaro", new JaroDistance());
|
||||
_distances.put("jaro-winkler", new JaroWinklerDistance());
|
||||
_distances.put("jaro-winkler-tfidf", new JaroWinklerTFIDFDistance());
|
||||
_distances.put("gzip", new GZipDistance());
|
||||
_distances.put("bzip2", new BZip2Distance());
|
||||
_distances.put("ppm", new PPMDistance());
|
||||
}
|
||||
|
||||
class VPTreeClusteringRowVisitor implements RowVisitor {
|
||||
|
||||
Distance _distance;
|
||||
JSONObject _config;
|
||||
VPTreeClusterer _clusterer;
|
||||
double _radius = 1.0f;
|
||||
|
||||
public VPTreeClusteringRowVisitor(Distance d, JSONObject o) {
|
||||
_distance = d;
|
||||
_config = o;
|
||||
_clusterer = new VPTreeClusterer(_distance);
|
||||
try {
|
||||
JSONObject params = o.getJSONObject("params");
|
||||
_radius = params.getDouble("radius");
|
||||
} catch (JSONException e) {
|
||||
//Gridworks.warn("No parameters found, using defaults");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
Cell cell = row.getCell(_colindex);
|
||||
if (cell != null && cell.value != null) {
|
||||
Object v = cell.value;
|
||||
String s = (v instanceof String) ? ((String) v) : v.toString();
|
||||
_clusterer.populate(s);
|
||||
count(s);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public List<Set<Serializable>> getClusters() {
|
||||
return _clusterer.getClusters(_radius);
|
||||
}
|
||||
}
|
||||
|
||||
class BlockingClusteringRowVisitor implements RowVisitor {
|
||||
|
||||
Distance _distance;
|
||||
JSONObject _config;
|
||||
double _radius = 1.0d;
|
||||
int _blockingNgramSize = 6;
|
||||
HashSet<String> _data;
|
||||
NGramClusterer _clusterer;
|
||||
|
||||
public BlockingClusteringRowVisitor(Distance d, JSONObject o) {
|
||||
_distance = d;
|
||||
_config = o;
|
||||
_data = new HashSet<String>();
|
||||
try {
|
||||
JSONObject params = o.getJSONObject("params");
|
||||
_radius = params.getDouble("radius");
|
||||
logger.debug("Use radius: {}", _radius);
|
||||
_blockingNgramSize = params.getInt("blocking-ngram-size");
|
||||
logger.debug("Use blocking ngram size: {}",_blockingNgramSize);
|
||||
} catch (JSONException e) {
|
||||
logger.debug("No parameters found, using defaults");
|
||||
}
|
||||
_clusterer = new NGramClusterer(_distance, _blockingNgramSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
Cell cell = row.getCell(_colindex);
|
||||
if (cell != null && cell.value != null) {
|
||||
Object v = cell.value;
|
||||
String s = (v instanceof String) ? ((String) v) : v.toString().intern();
|
||||
_clusterer.populate(s);
|
||||
count(s);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public List<Set<Serializable>> getClusters() {
|
||||
return _clusterer.getClusters(_radius);
|
||||
}
|
||||
}
|
||||
|
||||
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
|
||||
super.initializeFromJSON(project, o);
|
||||
_distance = _distances.get(o.getString("function").toLowerCase());
|
||||
}
|
||||
|
||||
public void computeClusters(Engine engine) {
|
||||
//VPTreeClusteringRowVisitor visitor = new VPTreeClusteringRowVisitor(_distance,_config);
|
||||
BlockingClusteringRowVisitor visitor = new BlockingClusteringRowVisitor(_distance,_config);
|
||||
FilteredRows filteredRows = engine.getAllFilteredRows();
|
||||
filteredRows.accept(_project, visitor);
|
||||
|
||||
_clusters = visitor.getClusters();
|
||||
}
|
||||
|
||||
public static class ValuesComparator implements Comparator<Entry<Serializable,Integer>>, Serializable {
|
||||
private static final long serialVersionUID = 204469656070583155L;
|
||||
public int compare(Entry<Serializable,Integer> o1, Entry<Serializable,Integer> o2) {
|
||||
return o2.getValue() - o1.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
writer.array();
|
||||
for (Set<Serializable> m : _clusters) {
|
||||
if (m.size() > 1) {
|
||||
Map<Serializable,Integer> internal_counts = new HashMap<Serializable,Integer>();
|
||||
for (Serializable s : m) {
|
||||
internal_counts.put(s,_counts.get(s));
|
||||
}
|
||||
List<Entry<Serializable,Integer>> values = new ArrayList<Entry<Serializable,Integer>>(internal_counts.entrySet());
|
||||
Collections.sort(values, new ValuesComparator());
|
||||
writer.array();
|
||||
for (Entry<Serializable,Integer> e : values) {
|
||||
writer.object();
|
||||
writer.key("v"); writer.value(e.getKey());
|
||||
writer.key("c"); writer.value(e.getValue());
|
||||
writer.endObject();
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
|
||||
private void count(Serializable s) {
|
||||
if (_counts.containsKey(s)) {
|
||||
_counts.put(s, _counts.get(s) + 1);
|
||||
} else {
|
||||
_counts.put(s, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,272 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.GridworksServlet;
|
||||
import com.google.gridworks.Jsonizable;
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.ProjectMetadata;
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.history.HistoryEntry;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.process.Process;
|
||||
import com.google.gridworks.util.ParsingUtilities;
|
||||
|
||||
/**
|
||||
* The super class of all calls that the client side can invoke, most of which
|
||||
* are AJAX calls.
|
||||
*/
|
||||
public abstract class Command {
|
||||
|
||||
final static protected Logger logger = LoggerFactory.getLogger("command");
|
||||
|
||||
protected GridworksServlet servlet;
|
||||
|
||||
public void init(GridworksServlet servlet) {
|
||||
this.servlet = servlet;
|
||||
}
|
||||
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
throw new UnsupportedOperationException();
|
||||
};
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
throw new UnsupportedOperationException();
|
||||
};
|
||||
|
||||
/**
|
||||
* Utility function to get the browsing engine's configuration as a JSON object
|
||||
* from the "engine" request parameter, most often in the POST body.
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
* @throws JSONException
|
||||
*/
|
||||
static protected JSONObject getEngineConfig(HttpServletRequest request)
|
||||
throws JSONException {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
|
||||
String json = request.getParameter("engine");
|
||||
try{
|
||||
return (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
|
||||
} catch (JSONException e){
|
||||
logger.debug( json + " could not be parsed to JSON");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to reconstruct the browsing engine from the "engine" request parameter,
|
||||
* most often in the POST body.
|
||||
*
|
||||
* @param request
|
||||
* @param project
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
static protected Engine getEngine(HttpServletRequest request, Project project)
|
||||
throws Exception {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
if (project == null) throw new IllegalArgumentException("parameter 'project' should not be null");
|
||||
|
||||
Engine engine = new Engine(project);
|
||||
JSONObject o = getEngineConfig(request);
|
||||
if (o != null)
|
||||
engine.initializeFromJSON(o);
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method for retrieving the Project object having the ID specified
|
||||
* in the "project" URL parameter.
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
* @throws ServletException
|
||||
*/
|
||||
protected Project getProject(HttpServletRequest request) throws ServletException {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
try {
|
||||
Project p = ProjectManager.singleton.getProject(Long.parseLong(request.getParameter("project")));
|
||||
if (p != null) {
|
||||
return p;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
throw new ServletException("Can't find project: missing or bad URL parameter");
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method for retrieving the ProjectMetadata object having the ID specified
|
||||
* in the "project" URL parameter.
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
* @throws ServletException
|
||||
*/
|
||||
protected ProjectMetadata getProjectMetadata(HttpServletRequest request) throws ServletException {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
try {
|
||||
ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(Long.parseLong(request.getParameter("project")));
|
||||
if (pm != null) {
|
||||
return pm;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
throw new ServletException("Can't find project metadata: missing or bad URL parameter");
|
||||
}
|
||||
|
||||
static protected int getIntegerParameter(HttpServletRequest request, String name, int def) {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
try {
|
||||
return Integer.parseInt(request.getParameter(name));
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
static protected JSONObject getJsonParameter(HttpServletRequest request, String name) {
|
||||
if (request == null) throw new IllegalArgumentException("parameter 'request' should not be null");
|
||||
String value = request.getParameter(name);
|
||||
if (value != null) {
|
||||
try {
|
||||
return ParsingUtilities.evaluateJsonStringToObject(value);
|
||||
} catch (JSONException e) {
|
||||
logger.warn("error getting json parameter",e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static protected void performProcessAndRespond(
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response,
|
||||
Project project,
|
||||
Process process
|
||||
) throws Exception {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
HistoryEntry historyEntry = project.processManager.queueProcess(process);
|
||||
if (historyEntry != null) {
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
Properties options = new Properties();
|
||||
|
||||
writer.object();
|
||||
writer.key("code"); writer.value("ok");
|
||||
writer.key("historyEntry"); historyEntry.write(writer, options);
|
||||
writer.endObject();
|
||||
|
||||
w.flush();
|
||||
w.close();
|
||||
} else {
|
||||
respond(response, "{ \"code\" : \"pending\" }");
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, String content)
|
||||
throws IOException, ServletException {
|
||||
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
Writer w = response.getWriter();
|
||||
if (w != null) {
|
||||
w.write(content);
|
||||
w.flush();
|
||||
w.close();
|
||||
} else {
|
||||
throw new ServletException("response returned a null writer");
|
||||
}
|
||||
}
|
||||
|
||||
static protected void respond(HttpServletResponse response, String status, String message)
|
||||
throws IOException, JSONException {
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
writer.object();
|
||||
writer.key("status"); writer.value(status);
|
||||
writer.key("message"); writer.value(message);
|
||||
writer.endObject();
|
||||
w.flush();
|
||||
w.close();
|
||||
}
|
||||
|
||||
static protected void respondJSON(HttpServletResponse response, Jsonizable o)
|
||||
throws IOException, JSONException {
|
||||
|
||||
respondJSON(response, o, new Properties());
|
||||
}
|
||||
|
||||
static protected void respondJSON(
|
||||
HttpServletResponse response, Jsonizable o, Properties options)
|
||||
throws IOException, JSONException {
|
||||
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
Writer w = response.getWriter();
|
||||
JSONWriter writer = new JSONWriter(w);
|
||||
|
||||
o.write(writer, options);
|
||||
w.flush();
|
||||
w.close();
|
||||
}
|
||||
|
||||
static protected void respondException(HttpServletResponse response, Exception e)
|
||||
throws IOException, ServletException {
|
||||
|
||||
logger.warn("Exception caught", e);
|
||||
|
||||
if (response == null) {
|
||||
throw new ServletException("Response object can't be null");
|
||||
}
|
||||
|
||||
try {
|
||||
JSONObject o = new JSONObject();
|
||||
o.put("code", "error");
|
||||
o.put("message", e.getMessage());
|
||||
|
||||
StringWriter sw = new StringWriter();
|
||||
PrintWriter pw = new PrintWriter(sw);
|
||||
e.printStackTrace(pw);
|
||||
pw.flush();
|
||||
sw.flush();
|
||||
|
||||
o.put("stack", sw.toString());
|
||||
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
respond(response, o.toString());
|
||||
} catch (JSONException e1) {
|
||||
e.printStackTrace(response.getWriter());
|
||||
}
|
||||
}
|
||||
|
||||
static protected void redirect(HttpServletResponse response, String url) throws IOException {
|
||||
response.sendRedirect(url);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,56 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.process.Process;
|
||||
|
||||
/**
|
||||
* Convenient super class for commands that perform abstract operations on
|
||||
* only the filtered rows based on the faceted browsing engine's configuration
|
||||
* on the client side.
|
||||
*
|
||||
* The engine's configuration is passed over as a POST body parameter. It is
|
||||
* retrieved, de-serialized, and used to construct the abstract operation.
|
||||
* The operation is then used to construct a process. The process is then
|
||||
* queued for execution. If the process is not long running and there is no
|
||||
* other queued process, then it gets executed right away, resulting in some
|
||||
* change to the history. Otherwise, it is pending. The client side can
|
||||
* decide how to update its UI depending on whether the process is done or
|
||||
* still pending.
|
||||
*
|
||||
* Note that there are interactions on the client side that change only
|
||||
* individual cells or individual rows (such as starring one row or editing
|
||||
* the text of one cell). These interactions do not depend on the faceted
|
||||
* browsing engine's configuration, and so they don't invoke commands that
|
||||
* subclass this class. See AnnotateOneRowCommand and EditOneCellCommand as
|
||||
* examples.
|
||||
*/
|
||||
abstract public class EngineDependentCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
|
||||
AbstractOperation op = createOperation(project, request, getEngineConfig(request));
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
performProcessAndRespond(request, response, project, process);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
abstract protected AbstractOperation createOperation(
|
||||
Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception;
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.preference.PreferenceStore;
|
||||
|
||||
public class GetAllPreferencesCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
Project project = request.getParameter("project") != null ? getProject(request) : null;
|
||||
PreferenceStore ps = project != null ?
|
||||
project.getMetadata().getPreferenceStore() :
|
||||
ProjectManager.singleton.getPreferenceStore();
|
||||
|
||||
try {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
JSONWriter writer = new JSONWriter(response.getWriter());
|
||||
|
||||
writer.object();
|
||||
|
||||
for (String key : ps.getKeys()) {
|
||||
Object pref = ps.get(key);
|
||||
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
|
||||
writer.key(key);
|
||||
writer.value(pref);
|
||||
}
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
} catch (JSONException e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,54 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.preference.PreferenceStore;
|
||||
import com.google.gridworks.preference.TopList;
|
||||
|
||||
public class GetPreferenceCommand extends Command {
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
Project project = request.getParameter("project") != null ? getProject(request) : null;
|
||||
PreferenceStore ps = project != null ?
|
||||
project.getMetadata().getPreferenceStore() :
|
||||
ProjectManager.singleton.getPreferenceStore();
|
||||
|
||||
String prefName = request.getParameter("name");
|
||||
Object pref = ps.get(prefName);
|
||||
|
||||
try {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
JSONWriter writer = new JSONWriter(response.getWriter());
|
||||
|
||||
writer.object();
|
||||
writer.key("value");
|
||||
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
|
||||
writer.value(pref);
|
||||
} else if (pref instanceof TopList) {
|
||||
TopList tl = (TopList) pref;
|
||||
tl.write(writer, new Properties());
|
||||
} else {
|
||||
writer.value(pref.toString());
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
} catch (JSONException e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.io.FileProjectManager;
|
||||
|
||||
public class OpenWorkspaceDirCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
String serverName = request.getServerName();
|
||||
|
||||
if (!"127.0.0.1".equals(serverName) && !"localhost".equals(serverName)) {
|
||||
respond(response, "{ \"code\" : \"error\", \"message\" : \"Workspace directory can only be opened on the local machine where Gridworks is run.\" }");
|
||||
} else if (ProjectManager.singleton instanceof FileProjectManager) {
|
||||
File dir = ((FileProjectManager) ProjectManager.singleton).getWorkspaceDir();
|
||||
|
||||
Runtime.getRuntime().exec(
|
||||
"open .",
|
||||
new String[] {},
|
||||
dir
|
||||
);
|
||||
|
||||
respond(response, "{ \"code\" : \"ok\" }");
|
||||
} else {
|
||||
respond(response, "{ \"code\" : \"error\", \"message\" : \"Workspace is not stored on the file system.\" }");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
package com.google.gridworks.commands;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONTokener;
|
||||
|
||||
import com.google.gridworks.ProjectManager;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.preference.PreferenceStore;
|
||||
|
||||
public class SetPreferenceCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
Project project = request.getParameter("project") != null ? getProject(request) : null;
|
||||
PreferenceStore ps = project != null ?
|
||||
project.getMetadata().getPreferenceStore() :
|
||||
ProjectManager.singleton.getPreferenceStore();
|
||||
|
||||
String prefName = request.getParameter("name");
|
||||
String valueString = request.getParameter("value");
|
||||
|
||||
try {
|
||||
Object o = valueString == null ? null : new JSONTokener(valueString).nextValue();
|
||||
|
||||
ps.put(prefName, PreferenceStore.loadObject(o));
|
||||
|
||||
respond(response, "{ \"code\" : \"ok\" }");
|
||||
} catch (JSONException e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,134 +0,0 @@
|
||||
package com.google.gridworks.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import oauth.signpost.OAuthConsumer;
|
||||
import oauth.signpost.OAuthProvider;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.oauth.Credentials;
|
||||
import com.google.gridworks.oauth.OAuthUtilities;
|
||||
import com.google.gridworks.oauth.Provider;
|
||||
|
||||
public class AuthorizeCommand extends Command {
|
||||
|
||||
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
// get the provider from the request
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
try {
|
||||
|
||||
// see if the request comes with access credentials
|
||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||
|
||||
// prepare the continuation URL that the OAuth provider will redirect the user to
|
||||
// (we need to make sure this URL points back to this code or the dance will never complete)
|
||||
String callbackURL = getBaseURL(request,provider);
|
||||
|
||||
if (access_credentials == null) {
|
||||
// access credentials are not available so we need to check
|
||||
// to see at what stage of the OAuth dance we are
|
||||
|
||||
// get the request token credentials
|
||||
Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST);
|
||||
|
||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider);
|
||||
OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider);
|
||||
|
||||
if (request_credentials == null) {
|
||||
// no credentials were found, so let's start the dance
|
||||
|
||||
// get the request token
|
||||
|
||||
String url = pp.retrieveRequestToken(consumer, callbackURL);
|
||||
|
||||
request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
||||
|
||||
// and set them to that we can retrieve them later in the second part of the dance
|
||||
Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600);
|
||||
|
||||
// now redirect the user to the Authorize URL where she can authenticate against the
|
||||
// service provider and authorize us.
|
||||
// The provider will bounce the user back here for us to continue the dance.
|
||||
|
||||
response.sendRedirect(url);
|
||||
} else {
|
||||
// we are at the second stage of the dance, so we need need to obtain the access credentials now
|
||||
|
||||
// if we got here, it means that the user performed a valid authentication against the
|
||||
// service provider and authorized us, so now we can request more permanent credentials
|
||||
// to the service provider and save those as well for later use.
|
||||
|
||||
// this is set only for OAuth 1.0a
|
||||
String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM);
|
||||
|
||||
pp.retrieveAccessToken(consumer, verificationCode);
|
||||
|
||||
access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
||||
|
||||
// no matter the result, we need to remove the request token
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
|
||||
Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600);
|
||||
|
||||
finish(response);
|
||||
}
|
||||
} else {
|
||||
finish(response);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void finish(HttpServletResponse response) throws IOException {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "text/html");
|
||||
|
||||
PrintWriter writer = response.getWriter();
|
||||
writer.write(
|
||||
"<html>" +
|
||||
"<body></body>" +
|
||||
"<script type='text/javascript'>" +
|
||||
"if (top.opener && top.opener.onauthorization) {" +
|
||||
" top.opener.onauthorization(window);" +
|
||||
"}" +
|
||||
"self.close();" +
|
||||
"</script>" +
|
||||
"</html>"
|
||||
);
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
private String getBaseURL(HttpServletRequest request, Provider provider) {
|
||||
String host = request.getHeader("host");
|
||||
if (host == null) {
|
||||
String referrer = request.getHeader("referer");
|
||||
if (referrer != null) {
|
||||
URI url;
|
||||
try {
|
||||
url = new URI(referrer);
|
||||
int port = url.getPort();
|
||||
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
|
||||
} catch (URISyntaxException e) {
|
||||
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL gridworks is listening to.");
|
||||
}
|
||||
}
|
||||
return "http://" + host + "/command/core/authorize/" + provider.getHost();
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
package com.google.gridworks.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.oauth.Credentials;
|
||||
import com.google.gridworks.oauth.OAuthUtilities;
|
||||
import com.google.gridworks.oauth.Provider;
|
||||
import com.google.gridworks.util.FreebaseUtils;
|
||||
|
||||
public class CheckAuthorizationCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
// this cookie should not be there, but this is good hygiene practice
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
|
||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
if (access_credentials != null) {
|
||||
String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
|
||||
response.getWriter().write(user_info);
|
||||
} else {
|
||||
respond(response, "401 Unauthorized", "You don't have the right credentials");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.info("error",e);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
package com.google.gridworks.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.oauth.Credentials;
|
||||
import com.google.gridworks.oauth.OAuthUtilities;
|
||||
import com.google.gridworks.oauth.Provider;
|
||||
|
||||
public class DeAuthorizeCommand extends Command {
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
||||
|
||||
respond(response, "200 OK", "");
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
package com.google.gridworks.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.oauth.OAuthUtilities;
|
||||
import com.google.gridworks.oauth.Provider;
|
||||
import com.google.gridworks.util.FreebaseUtils;
|
||||
|
||||
public class GetUserBadgesCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
String user_id = request.getParameter("user_id");
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
|
||||
response.getWriter().write(user_badges);
|
||||
} catch (Exception e) {
|
||||
logger.info("error",e);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
package com.google.gridworks.commands.browsing;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.clustering.Clusterer;
|
||||
import com.google.gridworks.clustering.binning.BinningClusterer;
|
||||
import com.google.gridworks.clustering.knn.kNNClusterer;
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class ComputeClustersCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("compute-clusters_command");
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
long start = System.currentTimeMillis();
|
||||
Project project = getProject(request);
|
||||
Engine engine = getEngine(request, project);
|
||||
JSONObject clusterer_conf = getJsonParameter(request,"clusterer");
|
||||
|
||||
Clusterer clusterer = null;
|
||||
String type = clusterer_conf.has("type") ? clusterer_conf.getString("type") : "binning";
|
||||
|
||||
if ("knn".equals(type)) {
|
||||
clusterer = new kNNClusterer();
|
||||
} else {
|
||||
clusterer = new BinningClusterer();
|
||||
}
|
||||
|
||||
clusterer.initializeFromJSON(project, clusterer_conf);
|
||||
|
||||
clusterer.computeClusters(engine);
|
||||
|
||||
respondJSON(response, clusterer);
|
||||
logger.info("computed clusters [{},{}] in {}ms", new Object[] { type, clusterer_conf.getString("function"), Long.toString(System.currentTimeMillis() - start) });
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
package com.google.gridworks.commands.browsing;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class ComputeFacetsCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
Engine engine = getEngine(request, project);
|
||||
|
||||
engine.computeFacets();
|
||||
|
||||
respondJSON(response, engine);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,175 +0,0 @@
|
||||
package com.google.gridworks.commands.browsing;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.ServletOutputStream;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gridworks.browsing.Engine;
|
||||
import com.google.gridworks.browsing.FilteredRows;
|
||||
import com.google.gridworks.browsing.facets.ScatterplotDrawingRowVisitor;
|
||||
import com.google.gridworks.browsing.facets.ScatterplotFacet;
|
||||
import com.google.gridworks.browsing.util.NumericBinIndex;
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.expr.Evaluable;
|
||||
import com.google.gridworks.expr.MetaParser;
|
||||
import com.google.gridworks.expr.ParsingException;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
|
||||
public class GetScatterplotCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("get-scatterplot_command");
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
Project project = getProject(request);
|
||||
Engine engine = getEngine(request, project);
|
||||
JSONObject conf = getJsonParameter(request,"plotter");
|
||||
|
||||
response.setHeader("Content-Type", "image/png");
|
||||
|
||||
ServletOutputStream sos = null;
|
||||
|
||||
try {
|
||||
sos = response.getOutputStream();
|
||||
draw(sos, project, engine, conf);
|
||||
} finally {
|
||||
sos.close();
|
||||
}
|
||||
|
||||
logger.trace("Drawn scatterplot in {} ms", Long.toString(System.currentTimeMillis() - start));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
public void draw(OutputStream output, Project project, Engine engine, JSONObject o) throws IOException, JSONException {
|
||||
|
||||
double min_x = 0;
|
||||
double min_y = 0;
|
||||
double max_x = 0;
|
||||
double max_y = 0;
|
||||
|
||||
int columnIndex_x = 0;
|
||||
int columnIndex_y = 0;
|
||||
|
||||
Evaluable eval_x = null;
|
||||
Evaluable eval_y = null;
|
||||
|
||||
int size = (o.has(ScatterplotFacet.SIZE)) ? o.getInt(ScatterplotFacet.SIZE) : 100;
|
||||
|
||||
double dot = (o.has(ScatterplotFacet.DOT)) ? o.getDouble(ScatterplotFacet.DOT) : 100;
|
||||
|
||||
int dim_x = (o.has(ScatterplotFacet.DIM_X)) ? ScatterplotFacet.getAxisDim(o.getString(ScatterplotFacet.DIM_X)) : ScatterplotFacet.LIN;
|
||||
int dim_y = (o.has(ScatterplotFacet.DIM_Y)) ? ScatterplotFacet.getAxisDim(o.getString(ScatterplotFacet.DIM_Y)) : ScatterplotFacet.LIN;
|
||||
|
||||
int rotation = (o.has(ScatterplotFacet.ROTATION)) ? ScatterplotFacet.getRotation(o.getString(ScatterplotFacet.ROTATION)) : ScatterplotFacet.NO_ROTATION;
|
||||
|
||||
String color_str = (o.has(ScatterplotFacet.COLOR)) ? o.getString(ScatterplotFacet.COLOR) : "000000";
|
||||
Color color = new Color(Integer.parseInt(color_str,16));
|
||||
|
||||
String base_color_str = (o.has(ScatterplotFacet.BASE_COLOR)) ? o.getString(ScatterplotFacet.BASE_COLOR) : null;
|
||||
Color base_color = base_color_str != null ? new Color(Integer.parseInt(base_color_str,16)) : null;
|
||||
|
||||
String columnName_x = o.getString(ScatterplotFacet.X_COLUMN_NAME);
|
||||
String expression_x = (o.has(ScatterplotFacet.X_EXPRESSION)) ? o.getString(ScatterplotFacet.X_EXPRESSION) : "value";
|
||||
|
||||
if (columnName_x.length() > 0) {
|
||||
Column x_column = project.columnModel.getColumnByName(columnName_x);
|
||||
if (x_column != null) {
|
||||
columnIndex_x = x_column.getCellIndex();
|
||||
}
|
||||
} else {
|
||||
columnIndex_x = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
eval_x = MetaParser.parse(expression_x);
|
||||
} catch (ParsingException e) {
|
||||
logger.warn("error parsing expression", e);
|
||||
}
|
||||
|
||||
String columnName_y = o.getString(ScatterplotFacet.Y_COLUMN_NAME);
|
||||
String expression_y = (o.has(ScatterplotFacet.Y_EXPRESSION)) ? o.getString(ScatterplotFacet.Y_EXPRESSION) : "value";
|
||||
|
||||
if (columnName_y.length() > 0) {
|
||||
Column y_column = project.columnModel.getColumnByName(columnName_y);
|
||||
if (y_column != null) {
|
||||
columnIndex_y = y_column.getCellIndex();
|
||||
}
|
||||
} else {
|
||||
columnIndex_y = -1;
|
||||
}
|
||||
|
||||
try {
|
||||
eval_y = MetaParser.parse(expression_y);
|
||||
} catch (ParsingException e) {
|
||||
logger.warn("error parsing expression", e);
|
||||
}
|
||||
|
||||
NumericBinIndex index_x = null;
|
||||
NumericBinIndex index_y = null;
|
||||
|
||||
String col_x_name = o.getString(ScatterplotFacet.X_COLUMN_NAME);
|
||||
Column column_x = project.columnModel.getColumnByName(col_x_name);
|
||||
if (column_x != null) {
|
||||
columnIndex_x = column_x.getCellIndex();
|
||||
index_x = ScatterplotFacet.getBinIndex(project, column_x, eval_x, expression_x);
|
||||
min_x = index_x.getMin();
|
||||
max_x = index_x.getMax();
|
||||
}
|
||||
|
||||
String col_y_name = o.getString(ScatterplotFacet.Y_COLUMN_NAME);
|
||||
Column column_y = project.columnModel.getColumnByName(col_y_name);
|
||||
if (column_y != null) {
|
||||
columnIndex_y = column_y.getCellIndex();
|
||||
index_y = ScatterplotFacet.getBinIndex(project, column_y, eval_y, expression_y);
|
||||
min_y = index_y.getMin();
|
||||
max_y = index_y.getMax();
|
||||
}
|
||||
|
||||
if (index_x != null && index_y != null && index_x.isNumeric() && index_y.isNumeric()) {
|
||||
ScatterplotDrawingRowVisitor drawer = new ScatterplotDrawingRowVisitor(
|
||||
columnIndex_x, columnIndex_y, min_x, max_x, min_y, max_y,
|
||||
size, dim_x, dim_y, rotation, dot, color
|
||||
);
|
||||
|
||||
if (base_color != null) {
|
||||
drawer.setColor(base_color);
|
||||
|
||||
FilteredRows filteredRows = engine.getAllRows();
|
||||
filteredRows.accept(project, drawer);
|
||||
|
||||
drawer.setColor(color);
|
||||
}
|
||||
|
||||
{
|
||||
FilteredRows filteredRows = engine.getAllFilteredRows();
|
||||
filteredRows.accept(project, drawer);
|
||||
}
|
||||
|
||||
ImageIO.write(drawer.getImage(), "png", output);
|
||||
} else {
|
||||
ImageIO.write(new BufferedImage(1, 1, BufferedImage.TYPE_4BYTE_ABGR), "png", output);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.commands.EngineDependentCommand;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.BlankDownOperation;
|
||||
|
||||
public class BlankDownCommand extends EngineDependentCommand {
|
||||
@Override
|
||||
protected AbstractOperation createOperation(Project project,
|
||||
HttpServletRequest request, JSONObject engineConfig) throws Exception {
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
|
||||
return new BlankDownOperation(
|
||||
engineConfig,
|
||||
columnName
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,128 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.history.Change;
|
||||
import com.google.gridworks.history.HistoryEntry;
|
||||
import com.google.gridworks.model.Cell;
|
||||
import com.google.gridworks.model.Column;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.model.changes.CellChange;
|
||||
import com.google.gridworks.process.QuickHistoryEntryProcess;
|
||||
import com.google.gridworks.util.ParsingUtilities;
|
||||
import com.google.gridworks.util.Pool;
|
||||
|
||||
public class EditOneCellCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
request.setCharacterEncoding("UTF-8");
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
|
||||
Project project = getProject(request);
|
||||
|
||||
int rowIndex = Integer.parseInt(request.getParameter("row"));
|
||||
int cellIndex = Integer.parseInt(request.getParameter("cell"));
|
||||
|
||||
String type = request.getParameter("type");
|
||||
String valueString = request.getParameter("value");
|
||||
Serializable value = null;
|
||||
|
||||
if ("number".equals(type)) {
|
||||
value = Double.parseDouble(valueString);
|
||||
} else if ("boolean".equals(type)) {
|
||||
value = "true".equalsIgnoreCase(valueString);
|
||||
} else if ("date".equals(type)) {
|
||||
value = ParsingUtilities.stringToDate(valueString);
|
||||
} else {
|
||||
value = valueString;
|
||||
}
|
||||
|
||||
EditOneCellProcess process = new EditOneCellProcess(
|
||||
project,
|
||||
"Edit single cell",
|
||||
rowIndex,
|
||||
cellIndex,
|
||||
value
|
||||
);
|
||||
|
||||
HistoryEntry historyEntry = project.processManager.queueProcess(process);
|
||||
if (historyEntry != null) {
|
||||
/*
|
||||
* If the operation has been done, return the new cell's data
|
||||
* so the client side can update the cell's rendering right away.
|
||||
*/
|
||||
JSONWriter writer = new JSONWriter(response.getWriter());
|
||||
|
||||
Pool pool = new Pool();
|
||||
Properties options = new Properties();
|
||||
options.put("pool", pool);
|
||||
|
||||
writer.object();
|
||||
writer.key("code"); writer.value("ok");
|
||||
writer.key("historyEntry"); historyEntry.write(writer, options);
|
||||
writer.key("cell"); process.newCell.write(writer, options);
|
||||
writer.key("pool"); pool.write(writer, options);
|
||||
writer.endObject();
|
||||
} else {
|
||||
respond(response, "{ \"code\" : \"pending\" }");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected static class EditOneCellProcess extends QuickHistoryEntryProcess {
|
||||
final int rowIndex;
|
||||
final int cellIndex;
|
||||
final Serializable value;
|
||||
Cell newCell;
|
||||
|
||||
EditOneCellProcess(
|
||||
Project project,
|
||||
String briefDescription,
|
||||
int rowIndex,
|
||||
int cellIndex,
|
||||
Serializable value
|
||||
) {
|
||||
super(project, briefDescription);
|
||||
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception {
|
||||
Cell cell = _project.rows.get(rowIndex).getCell(cellIndex);
|
||||
Column column = _project.columnModel.getColumnByCellIndex(cellIndex);
|
||||
if (column == null) {
|
||||
throw new Exception("No such column");
|
||||
}
|
||||
|
||||
newCell = new Cell(
|
||||
value,
|
||||
cell != null ? cell.recon : null
|
||||
);
|
||||
|
||||
String description =
|
||||
"Edit single cell on row " + (rowIndex + 1) +
|
||||
", column " + column.getName();
|
||||
|
||||
Change change = new CellChange(rowIndex, cellIndex, cell, newCell);
|
||||
|
||||
return new HistoryEntry(
|
||||
historyEntryID, _project, description, null, change);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.commands.EngineDependentCommand;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.FillDownOperation;
|
||||
|
||||
public class FillDownCommand extends EngineDependentCommand {
|
||||
@Override
|
||||
protected AbstractOperation createOperation(Project project,
|
||||
HttpServletRequest request, JSONObject engineConfig) throws Exception {
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
|
||||
return new FillDownOperation(
|
||||
engineConfig,
|
||||
columnName
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.MultiValuedCellJoinOperation;
|
||||
import com.google.gridworks.process.Process;
|
||||
|
||||
public class JoinMultiValueCellsCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
String keyColumnName = request.getParameter("keyColumnName");
|
||||
String separator = request.getParameter("separator");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellJoinOperation(columnName, keyColumnName, separator);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
performProcessAndRespond(request, response, project, process);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.commands.EngineDependentCommand;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.MassEditOperation;
|
||||
import com.google.gridworks.util.ParsingUtilities;
|
||||
|
||||
public class MassEditCommand extends EngineDependentCommand {
|
||||
@Override
|
||||
protected AbstractOperation createOperation(Project project,
|
||||
HttpServletRequest request, JSONObject engineConfig) throws Exception {
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
String expression = request.getParameter("expression");
|
||||
String editsString = request.getParameter("edits");
|
||||
|
||||
return new MassEditOperation(
|
||||
engineConfig,
|
||||
columnName,
|
||||
expression,
|
||||
MassEditOperation.reconstructEdits(ParsingUtilities.evaluateJsonStringToArray(editsString))
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.MultiValuedCellSplitOperation;
|
||||
import com.google.gridworks.process.Process;
|
||||
|
||||
public class SplitMultiValueCellsCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
String keyColumnName = request.getParameter("keyColumnName");
|
||||
String separator = request.getParameter("separator");
|
||||
String mode = request.getParameter("mode");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(columnName, keyColumnName, separator, mode);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
performProcessAndRespond(request, response, project, process);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.gridworks.commands.EngineDependentCommand;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.TextTransformOperation;
|
||||
|
||||
public class TextTransformCommand extends EngineDependentCommand {
|
||||
@Override
|
||||
protected AbstractOperation createOperation(Project project,
|
||||
HttpServletRequest request, JSONObject engineConfig) throws Exception {
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
String expression = request.getParameter("expression");
|
||||
String onError = request.getParameter("onError");
|
||||
boolean repeat = "true".equals(request.getParameter("repeat"));
|
||||
|
||||
int repeatCount = 10;
|
||||
String repeatCountString = request.getParameter("repeatCount");
|
||||
try {
|
||||
repeatCount = Math.max(Math.min(Integer.parseInt(repeatCountString), 10), 0);
|
||||
} catch (Exception e) {
|
||||
}
|
||||
|
||||
return new TextTransformOperation(
|
||||
engineConfig,
|
||||
columnName,
|
||||
expression,
|
||||
TextTransformOperation.stringToOnError(onError),
|
||||
repeat,
|
||||
repeatCount
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.TransposeColumnsIntoRowsOperation;
|
||||
import com.google.gridworks.process.Process;
|
||||
|
||||
public class TransposeColumnsIntoRowsCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
|
||||
String startColumnName = request.getParameter("startColumnName");
|
||||
int columnCount = Integer.parseInt(request.getParameter("columnCount"));
|
||||
String combinedColumnName = request.getParameter("combinedColumnName");
|
||||
|
||||
boolean prependColumnName = Boolean.parseBoolean(request.getParameter("prependColumnName"));
|
||||
String separator = request.getParameter("separator");
|
||||
boolean ignoreBlankCells = Boolean.parseBoolean(request.getParameter("ignoreBlankCells"));
|
||||
|
||||
AbstractOperation op = new TransposeColumnsIntoRowsOperation(
|
||||
startColumnName, columnCount, combinedColumnName, prependColumnName, separator, ignoreBlankCells);
|
||||
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
performProcessAndRespond(request, response, project, process);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
package com.google.gridworks.commands.cell;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.gridworks.commands.Command;
|
||||
import com.google.gridworks.model.AbstractOperation;
|
||||
import com.google.gridworks.model.Project;
|
||||
import com.google.gridworks.operations.cell.TransposeRowsIntoColumnsOperation;
|
||||
import com.google.gridworks.process.Process;
|
||||
|
||||
public class TransposeRowsIntoColumnsCommand extends Command {
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Project project = getProject(request);
|
||||
|
||||
String columnName = request.getParameter("columnName");
|
||||
int rowCount = Integer.parseInt(request.getParameter("rowCount"));
|
||||
|
||||
AbstractOperation op = new TransposeRowsIntoColumnsOperation(
|
||||
columnName, rowCount);
|
||||
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
performProcessAndRespond(request, response, project, process);
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user